rds-s3-backup 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p194@rds_s3_backup"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.16.20 ()" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+if [[ -s Gemfile ]] && {
+   ! builtin command -v bundle >/dev/null ||
+   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+}
+then
+  printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+  gem install bundler
+fi
+if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+then
+  bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+fi

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rds-s3-backup.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Tamara Temple
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Rds::S3::Backup
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rds-s3-backup'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rds-s3-backup
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/rds-s3-backup.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+=begin
+
+= rds-s3-backup-new.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Since*::        2013-02-25
+
+== Description
+
+Perform the nightly backup from production to S3, creating an obfuscated backup along the way
+
+=end
+
+require 'rubygems'
+require 'thor'
+
+require 'rds-s3-backup'
+
+require 'fog'
+
+require 'logger'
+require 'dogapi'
+
+class RdsS3Backup < Thor
+
+  desc "s3_dump", "Runs a mysqldump from a restored snapshot of the specified RDS instance, and uploads the dump to S3"
+  method_option :rds_instance_id
+  method_option :s3_bucket
+  method_option :backup_bucket
+  method_option :s3_prefix
+  method_option :aws_access_key_id
+  method_option :aws_secret_access_key
+  method_option :mysql_database
+  method_option :mysql_username
+  method_option :mysql_password
+  method_option :fog_timeout, :desc => 'Timeout in seconds for Fog requests (AWS connector)'
+  method_option :obfuscate_sql, :desc => 'Obfuscation Stored Procedure source'
+  method_option :dump_ttl, :desc => "Number of old dumps to keep."
+  method_option :dump_directory => "Where to store the temporary sql dump file."
+  method_option :config_file, :desc => "YAML file of defaults for any option. Options given during execution override these."
+  method_option :aws_region, :desc => "Region of your RDS server (and S3 storage, unless aws-s3-region is specified)."
+  method_option :aws_s3_region, :desc => "Region to store your S3 dumpfiles, if different from the RDS region"
+  method_option :db_subnet_group_name, :desc => "VPC RDS DB subnet"
+  method_option :db_instance_type, :desc => "DB Instance type"
+  method_option :instance_id, :desc => "Instance ID to create/mount EBS to"
+  method_option :log_level, :desc => "Set the level of verbosity. (debug|normal|warn|error|fatal)"
+
+  def s3_dump
+    thor_defaults = {
+      'backup_bucket' => 'novu-backups',
+      's3_prefix' => 'db_dumps',
+      'fog_timeout' => 30 * 60,
+      'obfuscate_sql' => '/usr/local/etc/obfuscate.sql',
+      'dump_directory' => '/mnt/secure',
+      'dump_ttl' => 0,
+      'aws_region' => 'us-east-1',
+      'aws_s3_region' => 'us-west-2',
+      'db_instance_type' => 'db.m1.small',
+      'log_level' => 'info'
+    }
+
+
+    Rds::S3::Backup.run(options,thor_defaults)
+  end
+
+end
+
+RdsS3Backup.start

data/lib/rds-s3-backup.rb

@@ -0,0 +1,132 @@
+require 'yaml'
+require 'logger'
+require "rds-s3-backup/version"
+require "rds-s3-backup/datadog"
+require "rds-s3-backup/myrds"
+require "rds-s3-backup/mys3"
+require "rds-s3-backup/mysqlcmds"
+require 'debugger'
+
+module Rds
+  module S3
+    module Backup
+      
+      def process_options(thor_options,thor_defaults)
+
+        options = thor_defaults
+
+        if thor_options[:config_file] && File.exists?(thor_options[:config_file])
+          begin
+            options = options.merge(YAML.load(File.read(thor_options[:config_file])))
+          rescue Exception => e
+            raise "Unable to read and parse #{thor_options[:config_file]}: #{e.class}: #{e}"
+          end
+        end
+
+        options.merge!(thor_options)
+
+        # Check for required options
+        missing_options = %w{rds_instance_id s3_bucket aws_access_key_id aws_secret_access_key mysql_database mysql_username mysql_password}.reduce([]) {|a, o| a << o unless options.has_key?(o); a}
+
+        raise "Missing required options #{missing_options.inspect} in either configuration or command line" if missing_options.count > 0
+        
+        options['timestamp'] = Time.new.strftime('%Y-%m-%d-%H-%M-%S-%Z') #  => "2013-02-25-19-28-55-CST" 
+
+        options
+      end
+
+      def run(thor_options,thor_defaults)
+
+        @options = process_options(thor_options,thor_defaults)
+        $logger = Logger.new(STDOUT)
+        $logger.level = set_logger_level(@options["log_level"])
+
+        $dogger = DataDog.new(@options['data_dog_api_key'])
+
+        "Starting RDS-S3-Backup at #{@options['timestamp']}".tap do |t|
+          $logger.info t
+          $dogger.send t
+        end
+
+
+        $logger.debug "#{File.basename(__FILE__)}:#{__LINE__}: Running with options:"
+        debug_opts = @options.dup
+        debug_opts['aws_access_key_id'] = 'X'*10
+        debug_opts['aws_secret_access_key'] = 'Y'*15
+        debug_opts['mysql_password'] = "ZZY"*5
+        debug_opts['data_dog_api_key'] = 'XYZZY'*3
+        $logger.debug debug_opts.to_yaml
+
+        begin
+
+          $logger.info "Creating RDS and S3 Connections"
+          rds = MyRDS.new(@options)
+          s3 = MyS3.new(@options)
+
+          $logger.info "Restoring Database"
+          rds.restore_db()
+
+          $logger.info "Dumping and saving original database contents"
+          real_data_file = "#{rds.server.id}-mysqldump-#{@options['timestamp']}.sql.gz"
+          s3.save_production(rds.dump(real_data_file))
+
+          if @options['dump_ttl'] > 0
+            $logger.info "Pruning old dumps"
+            s3.prune_files(:prefix => "#{rds.server.id}-mysqldump-",
+                           :keep => @options['dump_ttl'])
+          end
+
+          $logger.info "Obfuscating database"
+          rds.obfuscate()
+
+          $logger.info "Dumping and saving obfuscated database contents"
+          clean_data_file = "clean-mysqldump.sql.gz" 
+          s3.save_clean(rds.dump(clean_data_file))
+          
+        rescue Exception => e
+          msg = "ERROR in #{File.basename(__FILE__)}: #{e.class}: #{e}\n"
+          msg << e.backtrace.join("\n")
+          $logger.error msg
+          $dogger.send msg unless e.is_a?(DataDogException) # don't make a loop!
+          raise e
+        ensure
+          cleanup(rds, s3, [ real_data_file, clean_data_file ])
+        end
+        
+        "RDS-S3-Backup Completed Successfully".tap do |t|
+          $logger.info t
+          $dogger.send t, :alert_type => 'Info', :priority => 'low'
+        end
+
+        0                           # exit code 0
+
+
+      end
+
+      def cleanup(rds, s3, unlink_files=[])
+        rds.destroy() unless rds.nil?
+        s3.destroy() unless s3.nil?
+        
+        unlink_files = [ unlink_files ] unless unlink_files.is_a?(Array)
+        unlink_files = unlink_files.flatten
+        unlink_files.each {|f| File.unlink(f) if !f.nil? && File.exists?(f) }
+
+      end
+
+      def set_logger_level(ll)
+        case ll.downcase
+        when 'debug' ; Logger::DEBUG
+        when 'info' ; Logger::INFO
+        when 'warn' ; Logger::WARN
+        when 'error' ; Logger::ERROR
+        when 'fatal' ; Logger::FATAL
+        else
+          Logger::INFO
+        end
+      end
+
+      module_function :process_options, :run, :cleanup, :set_logger_level
+
+    end
+  end
+end

data/lib/rds-s3-backup/datadog.rb

@@ -0,0 +1,64 @@
+=begin
+
+= datadog.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-02-26
+*License*::      GPLv3
+*Version*::      0.0.1
+
+== Description
+
+=end
+
+module Rds
+  module S3
+    module Backup
+      class DataDogException < RuntimeError ; end
+      class DataDog
+
+        def initialize(api_key)
+          raise DataDogException.new("No DataDog API given") if api_key.nil? || api_key.empty?
+          @api_key = api_key
+        end
+
+        def make_dog_client()
+          begin
+            Dogapi::Client.new(@api_key)
+          rescue Exception => e
+            unless e.is_a?(DataDogException) # no loops!
+              raise DataDogException.new "Error setting up DataDog connection: #{e.class}: #{e}"
+            end
+          end
+        end
+
+
+        def send(msg, o={})
+          return                # do nothing for testing
+
+          return if msg.nil? || msg.empty?
+
+          options = {
+            :msg_title => 'Alert',
+            :alert_type => 'Error',
+            :tags => [ "host:#{`hostname`}", "env:production" ],
+            :priority => 'normal',
+            :source => 'My Apps'}.merge(o)
+
+          @dog_client ||= make_dog_client()
+
+          begin
+            @dog_client.emit_event(Dogapi::Event.new(msg, options))
+          rescue Exception => e
+            unless e.is_a?(DataDogException) # no loops!
+              raise DataDogException.new "Error sending #{msg} to DataDog with options #{options.inspect}: #{e.class}: #{e.message}"
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/rds-s3-backup/myrds.rb

@@ -0,0 +1,153 @@
+=begin
+
+= rds.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-02-26
+*License*::      GPLv3
+*Version*::      0.0.1
+
+== Description
+
+=end
+
+require 'fog'
+
+module Rds::S3::Backup
+
+
+  class MyRDSException < RuntimeError ; end
+  class MyRDS
+
+    attr_accessor :rds, :server
+
+    def initialize(opts)
+      @opts = opts
+
+      @rds = get_rds_connection(:aws_access_key_id => @opts['aws_access_key_id'],
+                                :aws_secret_access_key => @opts['aws_secret_access_key'],
+                                :region => @opts['aws_region'])
+
+      @server = get_rds_server(@opts['rds_instance_id'])
+
+    end
+
+    def restore_db
+
+      begin
+        @rds.restore_db_instance_from_db_snapshot(new_snap.id,
+                                                  backup_server_id,
+                                                  {"DBSubnetGroupName" => @opts['db_subnet_group_name'],
+                                                    "DBInstanceClass" => @opts['db_instance_type'] } )
+      rescue Exception => e
+        raise MyRDSException.new("Error in #{self.class}:restore_db: #{e.class}: #{e}")
+      end
+
+    end
+
+
+    def dump(backup_file_name)
+      @mysqlcmds ||= ::Rds::S3::Backup::MySqlCmds.new(backup_server.endpoint['Address'],
+                                                      @opts['mysql_username'],
+                                                      @opts['mysql_password'],
+                                                      @opts['mysql_database'])
+
+
+
+
+      @mysqlcmds.dump(backup_file_path(backup_file_name)) # returns the dump file path
+    end
+
+    def obfuscate
+      @mysqlcmds ||= ::Rds::S3::Backup::MySqlCmds.new(backup_server.endpoint['Address'],
+                                                      @opts['mysql_username'],
+                                                      @opts['mysql_password'],
+                                                      @opts['mysql_database'])
+
+
+
+
+      @mysqlcmds.exec(@opts['obfuscate_sql'])
+    end
+
+    def backup_file_path(backup_file_name)
+      File.join(@opts['dump_directory'], backup_file_name)    
+    end
+
+    def backup_server_id
+      @backup_server_id ||= "#{@server.id}-s3-dump-server-#{@opts['timestamp']}"
+    end
+
+    def new_snap
+      unless @new_snap
+        self.server.snapshots.new(:id => snap_name).save
+        @new_snap = self.server.snapshots.get(snap_name)
+
+        (0..1).each {|i|  $logger.debug "#{__FILE__}:#{__LINE__}: Waiting for new_snap server to be ready: #{i}"; @new_snap.wait_for { ready? } } # ready? sometimes lies
+
+      end
+
+      @new_snap
+    end
+
+    def backup_server
+      @backup_server ||= get_rds_server(backup_server_id)
+
+      (0..1).each {|i|  $logger.debug "#{__FILE__}:#{__LINE__}: Waiting for backup_server to be ready: #{i}" ;  @backup_server.wait_for {ready? } } # won't get fooled again!
+      @backup_server
+
+    end
+
+    def snap_name
+      @snap_name ||= "s3-dump-snap-#{@opts['timestamp']}"
+    end
+
+    def get_rds_connection(opts={})
+      options = {
+        :aws_access_key_id => @opts['aws_access_key_id'],
+        :aws_secret_access_key => @opts['aws_secret_access_key'],
+        :region => @opts['aws_region']}.merge(opts)
+      Fog.timeout=@opts['fog_timeout']
+      begin
+        connection = Fog::AWS::RDS.new(options)
+      rescue Exception => e
+        raise MyRDSException.new("Error in #{self.class}#get_rds_connection: #{e.class}: #{e}")
+      end
+      
+      raise MyRDSException.new("Unable to make RDS connection") if connection.nil?
+      connection
+
+
+    end
+
+    def get_rds_server(id)
+      begin
+        server = @rds.servers.get(id)
+      rescue Exception => e
+        raise MyRDSException.new("Error getting server in #{self.class}#get_rds_server: #{e.class}: #{e}")
+      end
+      raise MyRDSException.new("Server is nil for #{id}") if server.nil?
+      server
+    end
+
+
+    def destroy
+      
+      unless @new_snap.nil?
+        (0..1).each {|i| $logger.debug "#{__FILE__}:#{__LINE__}: Waiting for new_snap server to be ready in #{self.class}#destroy: #{i}"; @new_snap.wait_for { ready? } }
+        @new_snap.destroy
+      end
+      
+      unless @backup_server.nil?
+        (0..1).each {|i|  $logger.debug "#{__FILE__}:#{__LINE__}: Waiting for backup server to be ready in #{self.class}#destroy: #{i}"; @backup_server.wait_for {ready? } }
+        @backup_server.destroy(nil)
+      end
+
+    end
+
+
+
+  end
+
+end

data/lib/rds-s3-backup/mys3.rb

@@ -0,0 +1,128 @@
+=begin
+
+= mys3.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-02-26
+*License*::      GPLv3
+*Version*::      0.0.1
+
+== Description
+
+=end
+
+module Rds::S3::Backup
+  class MyS3Exception < RuntimeError ; end
+  class MyS3
+
+    def initialize(options)
+      @options = options
+
+      @s3 = get_storage(:aws_access_key_id => @options['aws_access_key_id'],
+                        :aws_secret_access_key => @options['aws_secret_access_key'],
+                        :region => @options['aws_s3_region'] ||= @options['aws_region'])
+    end
+
+    def get_storage(o={})
+      options = {
+        :aws_access_key_id => nil,
+        :aws_secret_access_key => nil,
+        :region => nil,
+        :provider => 'AWS',
+        :scheme => 'https'}.merge(o)
+      
+      begin
+        Fog::Storage.new(options)
+      rescue Exception => e
+        raise MyS3Exception.new "Error establishing storage connection: #{e.class}: #{e}"
+      end
+      
+    end
+
+    def get_bucket(bucket)
+
+      begin
+        @s3.directories.get(bucket)
+      rescue Exception => e
+        raise MyS3Exception.new "Error getting bucket #{bucket} in S3: #{e.class}: #{e}"
+      end
+
+    end
+
+    def save_production(file_path)
+      save(s3_bucket, file_path, :acl => 'private')
+    end
+
+    def save_clean(file_path)
+      save(backup_bucket, file_path, :acl => 'authenticated-read')
+    end
+
+
+
+    def save(bucket, file_path, o={})
+      options = {
+        :key => File.join(@options['s3_prefix'], File.basename(file_path)),
+        :body => File.open(file_path),
+        :acl => 'authenticated-read',
+        :encryption => 'AES256',
+        :content_type => 'application/x-gzip'}.merge(o)
+      
+      tries = 0
+      begin
+        
+        bucket.files.new(options).save
+
+      rescue Exception => e
+        if tries < 3
+          @logger.info "Retrying S3 upload after #{tries} tries"
+          tries += 1
+          retry
+        else
+          raise MyS3Exception.new "Could not save #{File.basename(file_path)} to S3 after 3 tries: #{e.class}: #{e}"
+        end
+      end
+
+    end
+
+
+    def s3_bucket
+      @s3_bucket ||= get_bucket(@options['s3_bucket'])
+    end
+
+    def backup_bucket
+      @backup_bucket ||= get_bucket(@options['backup_bucket'])
+    end
+
+
+    def prune_files(o={})
+      options = {
+        :prefix => '',
+        :keep => 1
+      }.merge(o)
+      
+      return if options[:keep] < 1 # must keep at least one, the last one!
+
+      my_files = s3_bucket.files.all('prefix' => options[:prefix])
+      return if my_files.nil?
+      
+      if my_files.count > options[:keep]
+        my_files.
+          sort {|x,y| x.last_modified <=> y.last_modified}.
+          take(files_by_date.count - options[:keep]).
+          each do |f|
+          logger.info "Deleting #{f.name}"
+          f.destroy
+        end
+      end
+    end
+
+    def destroy
+      # nothing really to do here...
+    end
+
+
+  end
+
+
+end

data/lib/rds-s3-backup/mysqlcmds.rb

@@ -0,0 +1,128 @@
+=begin
+
+= mysqldcmds.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-02-26
+*License*::      GPLv3
+*Version*::      0.0.1
+
+== Description
+
+=end
+
+require 'open3'
+
+module Rds::S3::Backup
+
+  class MySqlCmdsException < RuntimeError ; end
+  class MySqlCmds
+    attr_accessor :host, :user, :passwd, :dbname
+
+    def initialize(host, user, passwd, dbname)
+
+      # Assert real parameters
+      raise MySqlCmdsException.new("host is empty") if host.nil? || host.empty?
+      raise MySqlCmdsException.new("user is empty") if user.nil? || user.empty?
+      raise MySqlCmdsException.new("passwd is empty") if passwd.nil? || passwd.empty?
+      raise MySqlCmdsException.new("dbname is empty") if dbname.nil? || dbname.empty?
+      
+      # Store instance vars
+      @host = host
+      @user = user
+      @passwd = passwd
+      @dbname = dbname
+
+      # Find commands to execute
+      @mysqldump = `which mysqldump`.chomp
+      raise MySqlCmdsException.new("No mysqldump command found") if @mysqldump.empty?
+      @mysql = `which mysql`.chomp
+      raise MySqlCmdsException.new("No mysql command found") if @mysql.empty?
+      @gzip = `which gzip`.chomp
+      raise MySqlCmdsException.new("No gzip command found") if @gzip.empty?
+
+    end
+
+    def dump(output)
+      raise MySqlCmdsException.new("output is not specified") if output.nil? || output.empty?
+
+      dump_opts = ["--opt",
+                   "--add-drop-table",
+                   "--single-transaction",
+                   "--order-by-primary",
+                   "--host=#{@host}",
+                   "--user=#{@user}",
+                   "--password=XXPASSWORDXX",
+                   @dbname]
+
+      gzip_opts = ["--fast",
+                   "--stdout",
+                   "> #{output}"]
+
+
+      cmd = "#{@mysqldump} #{dump_opts.join(' ')} | #{@gzip} #{gzip_opts.join(' ')}"
+
+      run(cmd.gsub(/XXPASSWORDXX/,@passwd))
+      
+      output                      # return the dump file path
+
+    end
+
+    def exec(script)
+
+      return if script.nil? || script.empty?
+      raise MySqlCmdsException.new("#{script} is not readable!") unless File.readable?(script)
+
+      obf_opts = ["--host=#{@host}",
+                  "--user=#{user}",
+                  "--password=XXPASSWORDXX",
+                  @dbname]
+
+
+      cmd = "#{@mysql} #{obf_opts.join(' ')} < #{script}"
+
+      run(cmd.gsub(/XXPASSWORDXX/,@passwd))
+
+    end
+
+    def run(cmd)
+
+      result = _really_run_it(cmd)
+      if result.code != 0
+        error = "Mysql operation failed on  mysql://#{@user}@#{@host}/#{@dbname}: Error code: #{result.code}.\n"
+        error << "Command:\n#{cmd}\n"
+        error << "stdout:\n#{result.stdout.join("\n")}\n"
+        error << "stderr:\n#{result.stderr.join("\n")}\n"
+        raise MySqlCmdsException.new error
+      end
+      
+
+    end
+
+    Result = Struct.new :stdout, :stderr, :code
+
+    def _really_run_it(cmd)
+      result = Result.new
+      result.stdout = Array.new
+      result.stderr = Array.new
+      Open3.popen3(cmd) do |stdin, stdout, stderr, wait_thr|
+        stdin.close
+        until stdout.eof
+          result.stdout << stdout.gets.chomp
+        end 
+        stdout.close
+        until stderr.eof
+          result.stderr << stderr.gets.chomp
+        end
+        stderr.close
+        result.code = wait_thr.value.exitstatus
+      end
+      result
+    end
+
+  end
+
+
+
+end

data/lib/rds-s3-backup/version.rb

@@ -0,0 +1,7 @@
+module Rds
+  module S3
+    module Backup
+      VERSION = "0.0.1"
+    end
+  end
+end

data/rds-s3-backup.gemspec

@@ -0,0 +1,29 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rds-s3-backup/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "rds-s3-backup"
+  gem.version       = Rds::S3::Backup::VERSION
+  gem.authors       = ["Tamara Temple"]
+  gem.email         = ["tamouse@gmail.com"]
+  gem.description   = %q{"Backup from AWS RDS snapshot to AWS S3 as mysqldump"}
+  gem.summary       = %q{"Backup from AWS RDS snapshot to AWS S3 as mysqldump"}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency('thor')
+  gem.add_dependency('fog')
+  gem.add_dependency('logger')
+  gem.add_dependency('dogapi')
+  gem.add_development_dependency('rspec')
+  gem.add_development_dependency('aruba')
+  gem.add_development_dependency('cucumber')
+  gem.add_development_dependency('rake')
+  gem.add_development_dependency('debugger')
+  
+end

data/spec/rds-s3-backup_spec.rb

@@ -0,0 +1,85 @@
+=begin
+
+= rds-s3-backup_spec.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-02-26
+*License*::      GPLv3
+*Version*::      0.0.1
+
+== Description
+
+=end
+
+require 'rds-s3-backup'
+
+
+module Rds::S3::Backup
+
+  describe "has version" do
+    it {Rds::S3::Backup::VERSION.should == '0.0.1' }
+  end
+  
+  describe "module methods" do
+    
+    it { Rds::S3::Backup.should respond_to(:run) }
+    it { Rds::S3::Backup.should respond_to(:process_options) }
+    it { Rds::S3::Backup.should respond_to(:cleanup) }
+    it { Rds::S3::Backup.should respond_to(:set_logger_level) }
+  end
+
+  describe "#run method" do
+      let(:options) { {
+          :rds_instance_id => 'stagingdb',
+          :s3_bucket => 'novu-backups',
+          :backup_bucket => 'novu-backups',
+          :s3_prefix => 'db_dumps',
+          :aws_access_key_id => 'ABCDE',
+          :aws_secret_access_key => '0987654321',
+          :mysql_database => 'novu_text',
+          :mysql_username => 'novurun',
+          :mysql_password => 'passw0rd',
+          :obfuscate_sql => '/usr/local/etc/obfuscate.sql',
+          :dump_ttl => 0,
+          :dump_directory => '/mnt/secure',
+          :aws_region => 'us-east-1',
+          :aws_s3_region => 'us-west-2',
+          :db_subnet_group_name => 'staging db subnet',
+          :db_instance_type => 'db.m1.small',
+          :instance_id => '',
+          :log_level => 0,
+          :quiet => false,
+          :verbose => true
+        } }
+
+
+    before(:each) do
+      DataDog.stub(:new).and_return(OpenStruct.new(:send => true))
+
+      MyRDS.stub(:new).and_return(OpenStruct.new(:restore_db => true,
+                                                 :dump => Proc.new {|file| file },
+                                                 :obfuscate => true,
+                                                 :destroy => true,
+                                                 :server =>
+                                                 OpenStruct.new(:id => 1)
+                                                 ))
+      MyS3.stub(:new).and_return(OpenStruct.new(:save_production => true,
+                                                :save_clean => true,
+                                                :prune_files => true,
+                                                :destroy => true))
+    end
+
+    it "should process options" do
+      opts = Rds::S3::Backup.process_options(options)
+      opts.should be_a(Hash)
+      opts.should have_key(:timestamp)
+      opts[:timestamp].should_not be_nil
+    end
+
+
+    it { Rds::S3::Backup.run(options).should == 0 }
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

data/test_data/.gitignore

@@ -0,0 +1,2 @@
+obfuscate.sql
+config.yml

data/test_data/sample-config.yml

@@ -0,0 +1,12 @@
+rds_instance_id: 
+db_subnet_group_name: 
+s3_bucket: 
+aws_access_key_id: 
+aws_secret_access_key: 
+mysql_database: 
+mysql_username: 
+mysql_password: 
+dump_ttl: 0
+data_dog_api_key: 
+obfuscate_sql: test_data/obfuscate.sql
+dump_directory: test_output

metadata

@@ -0,0 +1,212 @@
+--- !ruby/object:Gem::Specification
+name: rds-s3-backup
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Tamara Temple
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dogapi
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! '"Backup from AWS RDS snapshot to AWS S3 as mysqldump"'
+email:
+- tamouse@gmail.com
+executables:
+- rds-s3-backup.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .rvmrc
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/rds-s3-backup.rb
+- lib/rds-s3-backup.rb
+- lib/rds-s3-backup/datadog.rb
+- lib/rds-s3-backup/myrds.rb
+- lib/rds-s3-backup/mys3.rb
+- lib/rds-s3-backup/mysqlcmds.rb
+- lib/rds-s3-backup/version.rb
+- rds-s3-backup.gemspec
+- spec/rds-s3-backup_spec.rb
+- spec/spec_helper.rb
+- test_data/.gitignore
+- test_data/sample-config.yml
+- test_output/.gitkeep
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: ! '"Backup from AWS RDS snapshot to AWS S3 as mysqldump"'
+test_files:
+- spec/rds-s3-backup_spec.rb
+- spec/spec_helper.rb