backups-cli 1.0.9

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3828b34e41c40252925145a874a3e7fd6532c840
+  data.tar.gz: 80c63eadf29e7635ae6f915e201778cd5d73a2a9
+SHA512:
+  metadata.gz: 82e51cd0ca2024470f35921864970bbb356208bbf0589a4d4dcb1f931bffeb51aca6be019f4da551c75abb1628d651ad147a06814048377969b13a11d58fefd1
+  data.tar.gz: 3a373562b5c61e704b8146173cd595d51de6b6f3848a100075913323e1d4f43b06271fdb4e1b7bc362656f99dad021666157169361ca9e490b0fbb12b3fcd286

data/.gitignore

@@ -0,0 +1,2 @@
+pkg/
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,5 @@
+---
+language: ruby
+
+script:
+- echo "Date is $(date)"

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2016 Schibsted Products & Technology AS.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,256 @@
+# backups-cli
+
+[![Build Status](https://travis-ci.org/schibsted/backups-cli.svg?branch=master)](https://travis-ci.org/schibsted/backups-cli)
+
+This tool backups different data sources to S3.
+
+
+# Usage
+
+To see all available commands run the tool with no options:
+
+```
+% ./bin/backups
+Commands:
+  backups crontab         # Shows the crontab config
+  backups help [COMMAND]  # Describe available commands or one specific command
+  backups install         # Sets up the crontab for all jobs
+  backups ls              # Lists all the configured jobs
+  backups show [JOB]      # Shows the merged config (for a JOB or them all)
+  backups start [JOB]     # Starts a backup JOB or all of them
+  backups verify JOB      # Restores and verifies a backup JOB
+  backups version         # Show current version
+
+```
+
+### Configuration
+
+The look up directories for the configs are, in order:
+
+  - The value in the `BACKUPS_CONFIG_DIR` env variable
+  - `${HOME}/.backups-cli`
+  - `/etc/backups-cli`
+
+The config directory will be the first it can find. All `*.yaml` files under
+that directory will be loaded. You might find useful to organise backups into
+subfolders. You can inline `BACKUPS_CONFIG_DIR` to override existing file system
+directories:
+
+    BACKUPS_CONFIG_DIR="/opt/my-backups" ./bin/backups start local-mysql
+
+
+### List all jobs
+
+To list all defined jobs call the `ls` command:
+
+```
+$ ./bin/backups ls
+JOB          CRONTAB    INSTALL  ENABLED
+local-mysql  0 4 * * *  true     true
+minimal      0 4 * * *  true     true
+```
+
+`INSTALL` refers if a job will be installed on the crontab and `ENABLED` if it
+can be run. You can run manually not installed jobs but the script will refuse
+to start disabled jobs.
+
+
+### Show one job details
+
+Use the command `show` to print out json details about all the details for a
+job. Note that the details presented here show the job as perceived from the
+script, meaning when all the defaults have been merged. Leave the `job` argument
+out to show the whole config after the app has processed all yaml files and
+applied the defaults. The merge pattern is:
+
+- Deep merge any local `default` settings into all jobs entries in the same file
+- Finally deep merge the top `default` entry (in the `main.yaml` file)
+
+These merges mean you can provide global and file fallbacks (local settings
+prevail), which means you can organise the backup config files by env/adapter
+type or any way you fancy.
+
+
+```json
+$ ./bin/backups show local-mysql | jq .
+{
+  "s3": {
+    "bucket": "acme-productions-backups",
+    "path": "local/mysql"
+  },
+  "encryption": {
+    "secret": "pass"
+  },
+  "tags": {
+    "env": "local",
+    "new": null
+  },
+  "type": "mysql",
+  "backup": {
+    "connection": {
+      "username": "root",
+      "password": "root"
+    }
+  },
+  "_name": "mysql-local",
+  "_file": "/Users/pedro/.backups-cli/local/mysql.yaml"
+}
+```
+
+### Add a new job
+
+Here's the minimal backup config needed:
+
+```yaml
+  ---
+  jobs:
+    my-fancy-backup:
+      type: mysql
+```
+
+Here's a more reasonable example of job (they go under the tops `jobs` entry):
+
+```yaml
+---
+jobs:
+  acme-productions:
+    type:             mysql
+    tags:
+      env:            local
+      type:           test
+    s3:
+      bucket:         acme-dumps
+      path:           production/mysql
+    backup:
+      connection:
+        host:         localhost
+        username:     backup
+        password:     password
+      crontab:
+        hour:         "*/4"
+```
+
+Note the lack of the leading and trailing slashes in the S3 path configs. The
+only required parameter is the `type`. All others parameters depend either on
+the adapter itself or are defaulted from the `backups.defaults` entry.
+
+Tags are metadata associated with a job and are passed to listeners, like
+Datadog or Slack. To remove a defaulted tag simply set it to `null`.
+
+
+### Run a job
+
+Run `./bin/backup` with the command `start` and pass a job name:
+
+    ./bin/backups start <job>
+
+The `<job>` parameter should exist somewhere on some yaml file within
+config directory under the entry `jobs`. Look into `config/jobs-example.yaml`
+for inspiration. For our example:
+
+    ./bin/backups start local-mysql
+
+The backups comes with a dry-run mode that you can use to preview what it would
+execute in a real run:
+
+    ./bin/backups start local-mysql --dry-run
+
+Backups are encrypted if you set the `encryption.secret` setting.
+
+Sending to S3 is done when the `s3.bucket` setting is set and the setting
+`s3.active` is not explicitly set to `false`.
+
+
+### Install the crontab
+
+Run the command `install` to loop through the configured job and set a crontab
+job for each of them
+
+    ./bin/backups install
+
+If you want to add a custom job but not install it you need to set the field
+`backup.crontan.install` to `false` in the crontab settings for that backup job:
+
+```yaml
+  acme-productions:
+    type:           mysql
+    enabled:        false
+    backup:
+      crontab:
+        install:    false
+        hour:       0
+```
+
+This config shows also where to disable the job. It's the `enabled` setting
+directly under the job.
+
+### Development
+
+Create the file `/etc/backups-cli/main.yaml`
+
+```yaml
+---
+backups:
+  paths:
+    backups:        /tmp/backups
+    verify:         /tmp/verify
+  listeners:
+    slack:
+      webhook:      https://hooks.slack.com/services/X/Y/Z
+      channel:      spt-payment-backups
+      _active:       false
+      _events:
+        - _start
+        - error
+    datadog:
+      api_key:      x
+      app_key:      y
+  crontab:
+    header:         "MAILTO=backups@spid.no\nPATH=/usr/bin:/bin:/usr/local/bin"
+
+defaults:
+  # options:
+  #   cleanup:        false
+  #   silent:         false
+  s3:
+    bucket:         740872874188-database-backups
+    path:           test/mysql
+    # active:         false
+  encryption:
+    secret:         x
+  backup:
+    crontab:
+      prefix:       ". /etc/profile.d/proxy.sh &&"
+      postfix:      "2>/dev/null"
+```
+
+
+### Release procedure
+
+To release a new version of this gem you need to have the Ryubygems credentials
+in place. Start with going into https://rubygems.org/profile/edit and run the
+`curl -u schibsted https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials`
+command they recommend there. This is the *only* step required to release a new
+gem version to rubygems.
+
+On this repo, just update the file `lib/backups/version.rb` with a bumped version
+and commit all changes to the git repo. Then run:
+
+    rake
+
+If you need to revoke an older gem version just run:
+
+    gem yank -v 0.5.14 backups-cli
+
+### Known issues
+
+The verify code runs two checks per table:
+
+- A `CHECK TABLE <table>`
+- A `SELECT COUNT(*) FROM <table>` on the table
+  
+The problem with the second is that the count is run after the dump and these
+stats can be skewed in inserts are done after the dump. If you start to get
+mismatches in this last one I suggest you use a float comparision, that is, one
+that only reports if the stats are more than say 1% different.
+  

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task default: [
+  :release,
+]

data/backups-cli.gemspec

@@ -0,0 +1,25 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+require "backups/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "backups-cli"
+  spec.version       = Backups::VERSION
+  spec.description   = "This tool backups different data sources to S3."
+  spec.summary       = "This tool backups different data sources to S3"
+  spec.authors       = ["Schibsted"]
+  spec.email         = ["spt@schibsted.com"]
+  spec.homepage      = "https://github.com/schibsted/backups-cli"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split $/
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename f }
+
+  spec.add_dependency "thor"
+  spec.add_dependency "json"
+  spec.add_dependency "mysql2"
+  spec.add_dependency "dogapi"
+  spec.add_dependency "slack-notifier"
+  spec.add_dependency "tablelize"
+  spec.add_development_dependency "rake"
+end

data/bin/backups

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+# Copyright 2016 Schibsted Products & Technology AS.
+# Licensed under the terms of the MIT license. See LICENSE in the project root.
+$:.push File.expand_path("../../lib", __FILE__)
+
+require "logger"
+require "backups"
+
+log_level = ENV.fetch("BACKUPS_LOG_LEVEL", "info").upcase
+DRY_RUN   = ENV.fetch("BACKUPS_DRY_RUN", "0") == "1"
+log_file  = ENV.fetch("BACKUPS_LOG_FILE", "/var/log/backups-cli.log")
+
+# Accept --debug or -x flags for stdout debug logging
+if ARGV.include? "--debug" or ARGV.include? "-x"
+  log_file = STDOUT
+  log_level = "DEBUG"
+  ARGV.delete("--debug")
+  ARGV.delete("-x")
+end
+
+$LOG_ACTIVE = 1
+$LOGGER = Logger.new(log_file)
+$LOGGER.level = Object.const_get("Logger::#{log_level}")
+# $LOGGER.progname = File.basename(__FILE__)
+$LOGGER.progname = "main"
+$LOGGER.formatter = proc do |type, time, name, message|
+  "[#{time}]  #{name}  #{type.ljust(6)}  #{message}\n"
+end
+
+$LOGGER.warn "Log level is #{log_level}"
+
+
+begin
+  Backups::Cli.start ARGV
+rescue => e
+  cmd = ARGV[0] || "undef"
+  db  = ARGV[1] || "undef"
+  msg = e.to_s.encode!("UTF-8", {undef: :replace})
+  Backups::Events.fire :error, {
+    error:   msg,
+    command: cmd,
+    db:      db,
+  }
+  $LOGGER.error "Error: #{msg}."
+
+  raise e
+end

data/lib/backups.rb

@@ -0,0 +1,19 @@
+# $:.push File.expand_path("../lib", __FILE__)
+
+require "backups/cli"
+require "backups/version"
+require "backups/events"
+require "backups/system"
+require "backups/loader"
+require "backups/runner"
+require "backups/crontab"
+require "backups/base"
+require "backups/driver/mysql"
+require "backups/stats/mysql"
+require "backups/adapter/mysql"
+require "backups/verify/mysql"
+require "backups/listener"
+require "backups/ext/hash"
+require "backups/ext/string"
+require "backups/ext/fixnum"
+require "backups/ext/nil_class"

data/lib/backups/adapter/mysql.rb

@@ -0,0 +1,202 @@
+require "yaml"
+
+module Backups
+  module Adapter
+    class Mysql < Base
+
+      include ::Backups::Stats::Mysql
+
+      def run
+        setup
+        create_dump
+        compress @job_dir, @job_zip, @secret
+
+        @job_size = File.size(@job_zip) if not $DRY_RUN
+        $LOGGER.info "File #{@job_zip} created with #{@job_size} bytes"
+
+        send_to_s3 @s3_bucket, @s3_prefix, @job_zip, @config if @s3_active
+        clean_up if @cleanup
+        report
+
+        return @report
+      end
+
+      def verify
+        s3     = @config.fetch("s3", {})
+        @name  = @config.fetch('_name')
+        bucket = s3.fetch("bucket", "")
+        path   = s3.fetch("path", "")
+        date   = get_date_path()
+        folder = "s3://#{bucket}/#{path}/#{@name}/#{date}/"
+        $LOGGER.warn "Latest s3 path is: " + get_latest_s3(folder)
+      end
+
+      private
+      # If you are lost here's what the vars mean:
+      #
+      # Variable    Value
+      # ---------   ----------------------------------
+      # backup_dir   /tmp/backups/
+      # source_dir     <source>/
+      # job_zip          <prefix>-<timestamp>.zip
+      # job_dir          <prefix>-<timestamp>/
+      def setup
+        load_configs
+
+        backups     = $GLOBAL.fetch('backups', {})
+        paths       = backups.fetch('paths', {})
+        encryption  = @config.fetch('encryption', {})
+        s3          = @config.fetch('s3', {})
+        prefix      = get_prefix()
+        date_path   = get_date_path()
+        timestamp   = get_timestamp()
+        backup_dir  = paths.fetch('backups', '/tmp/backups')
+
+        @backup     = @config.fetch('backup', {})
+        @options    = @config.fetch('options', {})
+        @connection = @backup.fetch('connection', {})
+        @secret     = encryption.fetch('secret', nil)
+        @cleanup    = @options.fetch('cleanup', true)
+        @name       = @config.fetch('_name')
+        @group      = @name.gsub(/[^0-9A-Za-z]/, '_')
+
+        job_name    = "#{prefix}-#{timestamp}"
+        @source_dir = "#{backup_dir}/#{@name}"
+        @job_dir    = "#{@source_dir}/#{job_name}"
+        @job_zip    = "#{@source_dir}/#{job_name}.zip"
+        @job_size   = nil
+
+        s3_path     = s3.fetch('path', "")
+        @s3_region  = s3.fetch('region', "eu-west-1")
+        @s3_bucket  = s3.fetch('bucket', nil)
+        @s3_active  = s3.fetch('active', @s3_bucket != nil)
+        @s3_prefix  = "#{s3_path}/#{@name}/#{date_path}"
+
+        @report     = {
+          started:   Time.now,
+          completed: nil,
+          file:      nil,
+          size:      nil,
+          report:    nil,
+        }
+      end
+
+      private
+      def get_dump_command database
+        host         = @connection.fetch("host", "localhost")
+        username     = @connection.fetch("username", nil)
+        password     = @connection.fetch("password", nil)
+        silent       = @options.fetch("silent", true)
+        master_data  = @options.fetch("master-data", 0)
+        disable_keys = @options.fetch("disable-keys", true)
+        lock_tables  = @options.fetch("lock-tables", false)
+        use_defaults = @group.size > 0 ? true : false
+        use_defaults = @options.fetch("use-defaults", use_defaults)
+        events       = @options.fetch("events", false)
+
+        command = []
+        command << "mysqldump"
+        command << "--defaults-group-suffix=_#{@group}"
+        command << "--host=#{host}"
+
+        command << "--user='#{username}'"           if username and !use_defaults
+        command << "--password='#{password}'"       if password and !use_defaults
+        command << "--master-data=#{master_data}"   if master_data
+        command << "--events"                       if events
+        command << "--skip-events"                  unless events
+        command << "--lock-tables"                  if lock_tables
+        command << "--single-transaction"           unless lock_tables
+        command << "--disable-keys"                 if disable_keys
+        command << database
+        command << "> #{@job_dir}/#{database}.sql"
+        command << "2>/dev/null"                    if silent
+
+        command.join(" ")
+      end
+
+      def get_db_list
+        host = @connection.fetch("host", "localhost")
+        skip = [
+          "information_schema",
+          "performance_schema",
+          "sys",
+        ]
+
+        command =  "mysql --defaults-group-suffix=_#{@group}"
+        command << " --host #{host}"
+        command << " -e 'show databases' | awk 'NR>1'"
+        $LOGGER.debug command
+
+        `#{command}`.split().select do |db|
+          not skip.include? db
+        end
+      end
+
+      def create_dump
+        $LOGGER.info "Creating #{@job_dir}"
+        mkdir @job_dir
+
+        database  = @backup.fetch("database", nil)
+        databases = @backup.fetch("databases", [])
+        databases << database if database
+
+        # Are we dumping specific databases
+        if databases.size > 0
+          $LOGGER.info "Preparing databases dump"
+          databases.each do |db|
+            create_database_dump db
+          end
+        else
+          create_server_dump
+        end
+      end
+
+      def create_server_dump
+        $LOGGER.info "Preparing server dump"
+        get_db_list().each do |db|
+          create_database_dump db
+        end
+      end
+
+      def create_database_dump db
+        $LOGGER.info "Dumping database #{db}"
+        command = get_dump_command(db)
+        create_database_stats db
+        exec command
+      end
+
+      def create_database_stats db
+        $LOGGER.debug "Creating #{db} database stats"
+        file = "#{@job_dir}/#{db}-stats.yaml"
+        stats = get_database_stats(db)
+        write file, stats.to_yaml
+      end
+
+      def clean_up
+        $LOGGER.info "Cleaning #{@source_dir}"
+        delete @job_zip if File.exists? @job_zip
+        nuke_dir @job_dir
+
+        delete_dir @source_dir
+      end
+
+      def report
+        path = "#{@s3_bucket}/#{@s3_prefix}/#{File.basename(@job_zip)}"
+
+        @report[:completed] = Time.now
+        @report[:file]      = @job_zip
+        @report[:size]      = @job_size
+        @report[:url]       = "s3://#{path}"
+        @report[:view]      = "https://console.aws.amazon.com/s3/buckets" +
+                              "#{path}?region=#{@s3_region}"
+      end
+
+      def get_latest_s3 folder
+        $LOGGER.info "Finding latest dump in folder #{folder}"
+        $LOGGER.info "aws s3 ls #{folder}/|awk '{ print $4 }'|tail -n 1"
+        exec "aws s3 ls #{folder}/|awk '{ print $4 }'|tail -n 1"
+      end
+
+    end
+  end
+end