rds_backup 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 72effc5ca22b06b2b27abaab144db3418a747193
+  data.tar.gz: 8e44fc0fe5edea2e7ba6dcc03c607e093287507b
+SHA512:
+  metadata.gz: 3807fb41057c41f7b8c8490258f29035c868c2965c06b8f1efb9d3c820464b2fbcb893960e3a7886b720d6070b8567e0a668d0bd6977baed97168d7c6646d364
+  data.tar.gz: be97982df38a7589e76ffe9aeb26dd1bd2ed889074cd910c5105d2de5093801d5732fe77d2beefb525adbf010040d6662d6e04f192a75c86dfd47c78c1af6a13

data/.gitignore

@@ -0,0 +1,38 @@
+# Project specific
+config.yml
+
+# Ruby
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rubocop.yml

@@ -0,0 +1,2 @@
+Documentation:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.1.2

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,75 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (1.51.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    builder (3.2.2)
+    byebug (2.7.0)
+      columnize (~> 0.3)
+      debugger-linecache (~> 1.2)
+    coderay (1.1.0)
+    columnize (0.8.9)
+    debugger-linecache (1.2.0)
+    excon (0.39.5)
+    fog (1.23.0)
+      fog-brightbox
+      fog-core (~> 1.23)
+      fog-json
+      fog-softlayer
+      ipaddress (~> 0.5)
+      nokogiri (~> 1.5, >= 1.5.11)
+    fog-brightbox (0.3.0)
+      fog-core (~> 1.22)
+      fog-json
+      inflecto
+    fog-core (1.23.0)
+      builder
+      excon (~> 0.38)
+      formatador (~> 0.2)
+      mime-types
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+    fog-json (1.0.0)
+      multi_json (~> 1.0)
+    fog-softlayer (0.3.14)
+      fog-core
+      fog-json
+    formatador (0.2.5)
+    hipchat (1.3.0)
+      httparty
+    httparty (0.13.1)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    inflecto (0.0.2)
+    ipaddress (0.8.0)
+    json (1.8.1)
+    method_source (0.8.2)
+    mime-types (2.3)
+    mini_portile (0.6.0)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.1)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (1.3.3)
+      byebug (~> 2.7)
+      pry (~> 0.10)
+    slop (3.6.0)
+    yell (2.0.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk
+  fog
+  hipchat
+  pry-byebug
+  yell

data/LICENSE.md

@@ -0,0 +1,22 @@
+Copyright (c) 2014 IDXP Analytics Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# RDS Backup
+
+This gem backs up RDS databases to different cloud providers, granting an extra level of mind peacefulness.
+
+## Installation
+
+`rds_backup` should be run as a command line tool, so it makes sense to install it globally
+
+```bash
+$ gem install rds_backup
+```
+
+## Usage
+
+`rds_backup` expects a `config.yml` file on the folder it is run, [here is a sample](config.yml.sample).
+
+Currently, you need a Google Cloud account with access to Cloud Storage. Sign up [here](https://developers.google.com/storage/docs/signup?csw=1) and get your credentials [here](https://storage.cloud.google.com/m) under the section "Interoperable Access".
+
+AWS credentials with access to SES and a Hipchat Room token from API v2 are also required at this time.
+
+Then, simply run the gem's binary:
+
+```bash
+$ rds_backup
+```
+
+### Cron Usage
+
+This tool is usually scheduled in cron, below is an example
+
+```
+0 3 * * * . $HOME/.profile; cd $HOME/rds_backup && rds_backup >> /var/log/rds_backup/rds_backup.log 2>&1
+```
+
+## Roadmap
+
+* Tests
+* Multiple cloud providers (we're already using fog)
+* More yell adapters (Slack, Campfire, IRC, etc)
+* More flexibe rules on backups to keep (à la [ec2-expire-snapshots](https://github.com/alestic/ec2-expire-snapshots))
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/bin/rds_backup

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'rds_backup'
+rescue LoadError
+  require 'rubygems'
+  require 'rds_backup'
+end
+
+rb = RdsBackup::Backup.new
+rb.backup

data/config.yml.sample

@@ -0,0 +1,25 @@
+cloud_storage:
+  access_key_id:
+  secret_access_key:
+mysql:
+  user:
+  password:
+  host:
+  database:
+  ssl: false
+  rds_cert_path: /home/user/rds_backup/mysql-ssl-ca-cert.pem
+aws:
+  access_key_id:
+  secret_access_key:
+email:
+  from_address: no-reply@example.com
+  to_addresses:
+    - git@example.com
+  subject: 'RDS Backup'
+hipchat:
+  token:
+  rooms:
+    - Test
+loggers:
+  - ses
+  - hipchat

data/lib/rds_backup.rb

@@ -0,0 +1,4 @@
+require 'rds_backup/backup'
+
+module RdsBackup
+end

data/lib/rds_backup/backup.rb

@@ -0,0 +1,123 @@
+require 'fog'
+require 'open3'
+require 'yaml'
+require_relative '../yell_adapters/ses_adapter'
+require_relative '../yell_adapters/hipchat_adapter'
+
+module RdsBackup
+  class Backup
+    def initialize(config_file = 'config.yml')
+      config = YAML.load(File.open(config_file))
+      config_google(config)
+      config_mysql(config)
+      config_logger(config)
+
+      @backups_to_keep = config['backups_to_keep'] || 3
+      current_utc_time = Time.now.getutc.strftime('%Y%m%d%H%M%S')
+      @file_name = "#{mysql_database}-#{current_utc_time}.sql.gz"
+    end
+
+    def backup
+      logger.info 'Starting...'
+      dump_database
+      upload_backup
+      prune_old_backups
+      remove_local_dump
+    ensure
+      logger.info 'Exiting...'
+      logger.close
+    end
+
+    private
+
+    attr_accessor :mysql_database, :mysql_host, :mysql_user, :mysql_password,
+                  :mysql_ssl, :rds_cert_path, :fog_directory, :file_name,
+                  :backups_to_keep, :logger
+
+    def config_mysql(config)
+      @mysql_database = config['mysql']['database']
+      @mysql_host = config['mysql']['host']
+      @mysql_user = config['mysql']['user']
+      @mysql_password = config['mysql']['password']
+      @mysql_ssl = config['mysql']['ssl']
+      @rds_cert_path = config['mysql']['rds_cert_path'] if @mysql_ssl
+    end
+
+    def config_google(config)
+      google_access = config['cloud_storage']['access_key_id']
+      google_secret = config['cloud_storage']['secret_access_key']
+      connection = Fog::Storage.new(
+        provider: 'Google',
+        google_storage_access_key_id: google_access,
+        google_storage_secret_access_key: google_secret
+      )
+      @fog_directory = connection.directories.get('idxp-rds-backup')
+    end
+
+    def config_logger(config)
+      @logger = Yell.new do |yell_logger|
+        config['loggers'].each do |logger|
+          send("config_#{logger}_adapter", yell_logger, config)
+        end
+      end
+    end
+
+    def config_ses_adapter(logger, config)
+      logger.adapter :ses_adapter,
+                     format: Yell::BasicFormat,
+                     aws_access_key_id: config['aws']['access_key_id'],
+                     aws_secret_access_key: config['aws']['secret_access_key'],
+                     email_config: config['email']
+    end
+
+    def config_hipchat_adapter(logger, config)
+      logger.adapter :hipchat_adapter,
+                     format: Yell::BasicFormat,
+                     hipchat_token: config['hipchat']['token'],
+                     hipchat_rooms: config['hipchat']['rooms']
+    end
+
+    def dump_database
+      _out, err, _status = Open3.capture3 mysqldump_cmd
+
+      if err.empty?
+        logger.info 'Database dump successfully created'
+      else
+        logger.error "Error when dumping the database: #{err.strip}"
+        remove_local_dump
+        fail RuntimeError
+      end
+    end
+
+    def mysqldump_cmd
+      cmd = "mysqldump -u#{mysql_user} "
+      cmd += "-p#{mysql_password} " if mysql_password
+      cmd += "--ssl_ca=#{rds_cert_path} " if mysql_ssl
+      cmd += '--single-transaction --routines --triggers '\
+             "-h #{mysql_host} #{mysql_database} "\
+             "| gzip -c > #{file_name}"
+      cmd
+    end
+
+    def upload_backup
+      fog_directory.files.create(key: file_name, body: File.open(file_name))
+      logger.info 'Backup uploaded to Google'
+    rescue
+      logger.error 'Error while uploading dump to Google'
+    end
+
+    def prune_old_backups
+      sorted_files = fog_directory.files.reload.sort do |x, y|
+        x.last_modified <=> y.last_modified
+      end
+      sorted_files[0 .. -backups_to_keep - 1].each { |f| f.destroy }
+      logger.info 'Old backups pruned'
+    rescue
+      logger.error 'Error while pruning old backups'
+    end
+
+    def remove_local_dump
+      Open3.capture3 "rm -rf #{file_name}"
+    end
+  end
+end

data/lib/rds_backup/version.rb

@@ -0,0 +1,3 @@
+module RdsBackup
+  VERSION = '0.1.0'
+end

data/lib/yell_adapters/hipchat_adapter.rb

@@ -0,0 +1,47 @@
+require 'yell'
+require 'hipchat'
+
+class HipchatAdapter < Yell::Adapters::Base
+  include Yell::Helpers::Base
+  include Yell::Helpers::Formatter
+
+  attr_accessor :hipchat, :body_text, :hipchat_rooms, :errors
+
+  setup do |options|
+    self.formatter = options[:format]
+    self.hipchat = HipChat::Client.new(
+      options[:hipchat_token], api_version: 'v2'
+    )
+    self.hipchat_rooms = options[:hipchat_rooms]
+    self.body_text = ''
+    self.errors = false
+  end
+
+  write do |event|
+    self.body_text += formatter.call(event)
+    self.errors = true if event.level >= 3
+  end
+
+  close do
+    format_body_text_to_html
+    color = errors? ? 'red' : 'green'
+    hipchat_rooms.each do |room|
+      hipchat[room].send('RDS Backup', body_text, message_format: 'html',
+                                                  color: color)
+    end
+  end
+
+  private
+
+  def errors?
+    errors
+  end
+
+  def format_body_text_to_html
+    body_text.gsub!("\n", '<br>')
+    prepend = errors? ? '<strong>Error!</strong>' : '<strong>Success!</strong>'
+    self.body_text = [prepend, body_text].join '<br>'
+  end
+end
+
+Yell::Adapters.register :hipchat_adapter, HipchatAdapter

data/lib/yell_adapters/ses_adapter.rb

@@ -0,0 +1,48 @@
+require 'yell'
+require 'aws-sdk'
+
+class SesAdapter < Yell::Adapters::Base
+  include Yell::Helpers::Base
+  include Yell::Helpers::Formatter
+
+  attr_accessor :ses, :body_text, :email_config, :errors
+
+  setup do |options|
+    self.formatter = options[:format]
+    self.ses = AWS::SimpleEmailService.new(
+      access_key_id: options[:aws_access_key_id],
+      secret_access_key: options[:aws_secret_access_key]
+    )
+    self.email_config = options[:email_config]
+    self.body_text = ''
+    self.errors = false
+  end
+
+  write do |event|
+    self.body_text += formatter.call(event)
+    self.errors = true if event.level >= 3
+  end
+
+  close do
+    base_subject = email_config['subject'] + Time.now.utc.strftime(' %Y/%m/%d')
+    subject = format_subject(base_subject)
+    ses.send_email(
+      subject: subject,
+      from: email_config['from_address'],
+      to: email_config['to_addresses'],
+      body_text: body_text
+    )
+  end
+
+  private
+
+  def errors?
+    errors
+  end
+
+  def format_subject(base_subject)
+    errors? ? "Error: #{base_subject}" : "Sucess: #{base_subject}"
+  end
+end
+
+Yell::Adapters.register :ses_adapter, SesAdapter

data/rds_backup.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rds_backup/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'rds_backup'
+  spec.version       = RdsBackup::VERSION
+  spec.authors       = ['Artur Rodrigues']
+  spec.email         = ['arturhoo@gmail.com']
+  spec.description   = 'Backs up RDS Databases and sync them to multiple '\
+                       'cloud providers'
+  spec.summary       = 'Offsite and offprovider backups for RDS Databases'
+  spec.homepage      = 'https://github.com/idxp/rds_backup'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(/^(test|spec|features)\//)
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry_byebug'
+
+  spec.add_dependency 'aws-sdk'
+  spec.add_dependency 'fog'
+  spec.add_dependency 'yell'
+  spec.add_dependency 'hipchat'
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: rds_backup
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Artur Rodrigues
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry_byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yell
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hipchat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Backs up RDS Databases and sync them to multiple cloud providers
+email:
+- arturhoo@gmail.com
+executables:
+- rds_backup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".ruby-version"
+- Gemfile
+- Gemfile.lock
+- LICENSE.md
+- README.md
+- Rakefile
+- bin/rds_backup
+- config.yml.sample
+- lib/rds_backup.rb
+- lib/rds_backup/backup.rb
+- lib/rds_backup/version.rb
+- lib/yell_adapters/hipchat_adapter.rb
+- lib/yell_adapters/ses_adapter.rb
+- rds_backup.gemspec
+homepage: https://github.com/idxp/rds_backup
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Offsite and offprovider backups for RDS Databases
+test_files: []