new_backup 1.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+.DS_Store
+/.rspec
+html
+pkg
+results.html
+tmp

data/.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p392@new_backup"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.16.20 ()" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in new_backup.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,82 @@
+PATH
+  remote: .
+  specs:
+    new_backup (1.0.1)
+      RunIt
+      dogapi
+      fog
+      methadone (~> 1.2.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    RunIt (1.0.1)
+    aruba (0.5.3)
+      childprocess (>= 0.3.6)
+      cucumber (>= 1.1.1)
+      rspec-expectations (>= 2.7.0)
+    builder (3.2.0)
+    childprocess (0.3.9)
+      ffi (~> 1.0, >= 1.0.11)
+    columnize (0.3.6)
+    cucumber (1.3.1)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.12.0)
+      multi_json (~> 1.3)
+    debugger (1.6.0)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.2.1)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.2.2)
+    diff-lcs (1.2.4)
+    dogapi (1.6.0)
+      json (>= 1.5.1)
+    excon (0.22.1)
+    ffi (1.8.1)
+    fog (1.11.1)
+      builder
+      excon (~> 0.20)
+      formatador (~> 0.2.0)
+      json (~> 1.7)
+      mime-types
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5.0)
+      ruby-hmac
+    formatador (0.2.4)
+    gherkin (2.12.0)
+      multi_json (~> 1.3)
+    json (1.8.0)
+    methadone (1.2.6)
+      bundler
+    mime-types (1.23)
+    multi_json (1.7.3)
+    net-scp (1.1.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.6.7)
+    nokogiri (1.5.9)
+    rake (0.9.6)
+    rdoc (4.0.1)
+      json (~> 1.4)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+    ruby-hmac (0.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aruba
+  debugger
+  new_backup!
+  rake (~> 0.9.2)
+  rdoc
+  rspec

data/LICENSE.txt

@@ -0,0 +1,7 @@
+Copyright (c) 2013 YOUR NAME 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# NewBackup
+
+Backs up an RDS instance, saving the backup in an S3 bucket.
+Runs an obfuscation script on the backup, saving that in an S3 bucket as "clean".
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'new_backup'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install new_backup
+
+## Usage
+
+    $ new_backup [options] config-file
+
+## Configuration
+
+The `config-file` on the command line needs to contain the following:
+
+    fog_timeout: (in seconds)
+    dump_directory: (directory to save files)
+    dump_ttl: (number of backups to keep)
+    aws_region: 
+    db_instance_type: 
+    timestamp_format: (strftime format string, no spaces for extra goodness)
+    rds_instance_id: (AWS RDS instance to back up)
+    db_subnet_group_name: (AWS RDS Subnet Group)
+    backup_bucket: (AWS S3 bucket to contain the raw backups)
+    s3_bucket: (AWS S3 bucket to contain the cleaned (obfuscated) backups)
+    s3_prefix: (Prefix, aka "subdirectory", to store backups in)
+    aws_s3_region: (AWS S3 region to use for saving)
+    mysql_username: (user name of data base)
+    mysql_database: (data base in RDS instance to back up)
+    mysql_password: (password for user)
+    obfuscate_script: (path to script to run to obfuscate the database)
+    datadog_apikey: (Datadog API key to use to report progress and errors)
+    aws_access_key_id: (Your AWS access key)
+    aws_secret_access_key: (Your AWS secret key)
+	
+The file `sample-config.yml` can be modified to your suiting.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/README.rdoc

@@ -0,0 +1,53 @@
+= new_backup - Backup and obfuscate an RDS instance, storing results in an S3 bucket
+
+Author::  Tamara Temple (tamara.temple@novu.com)
+Copyright:: Copyright (c) 2013 Novu, LLC
+
+License:: mit, see LICENSE.txt
+
+new_backup creates a snapshot of an RDS instance, and then dumps the
+snapshot to a file, compressing it, and saving that file to S3. It
+also runs a script to obfuscate the database snapshot, dumps and
+compresses the cleaned database, and saves that file to S3 as well.
+
+== Links
+
+* {Source on Github}[https://github.com/novu/new-backup]
+
+== Install
+
+To install, enter the following:
+
+    $ gem install new-backup
+
+This is a stand-alone utility, so it would probably be best to install
+this gem at a system-wide level.
+
+== Examples
+
+Run the backup using the info as specified in the file `config.yml`
+
+   $ new-backup config.yml
+
+Turn on debug-level logging:
+
+   $ new-backup --log-level debug config.yml
+
+For help, run:
+
+   $ new-backup --help
+
+
+== Configuration
+
+The `sample-config.yml` file contains the necessary elements to
+configure the script. Copy it and modify it so it meets your needs.
+
+
+== Contributing
+
+* Fork it
+* Create your feature branch (`git checkout -b my-new-feature`)
+* Commit your changes (`git commit -am 'Add some feature'`)
+* Push to the branch (`git push origin my-new-feature`)
+* Create new Pull Request

data/Rakefile

@@ -0,0 +1,70 @@
+def dump_load_path
+  puts $LOAD_PATH.join("\n")
+  found = nil
+  $LOAD_PATH.each do |path|
+    if File.exists?(File.join(path,"rspec"))
+      puts "Found rspec in #{path}"
+      if File.exists?(File.join(path,"rspec","core"))
+        puts "Found core"
+        if File.exists?(File.join(path,"rspec","core","rake_task"))
+          puts "Found rake_task"
+          found = path
+        else
+          puts "!! no rake_task"
+        end
+      else
+        puts "!!! no core"
+      end
+    end
+  end
+  if found.nil?
+    puts "Didn't find rspec/core/rake_task anywhere"
+  else
+    puts "Found in #{path}"
+  end
+end
+require 'bundler'
+require 'rake/clean'
+
+begin
+require 'rspec/core/rake_task'
+rescue LoadError
+dump_load_path
+raise
+end
+
+require 'cucumber'
+require 'cucumber/rake/task'
+gem 'rdoc' # we need the installed RDoc gem, not the system one
+require 'rdoc/task'
+
+include Rake::DSL
+
+Bundler::GemHelper.install_tasks
+
+
+RSpec::Core::RakeTask.new do |t|
+  # Put spec opts in a file named .rspec in root
+end
+
+
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty --no-source -x"
+  t.fork = false
+end
+
+Rake::RDocTask.new do |rd|
+  
+  rd.main = "README.rdoc"
+  
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+end
+
+task :default => [:spec,:features]
+
+desc 'Live testing to AWS test instances'
+task :live_test do |t, args|
+  sh "bundle exec bin/new_backup --log-level debug --debug tmp/livetest-config.yml"
+end

data/bin/new_backup

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'methadone'
+require 'yaml'
+require 'new_backup.rb'
+
+class App
+  include Methadone::Main
+  include Methadone::CLILogging
+
+  main do |config_file| # Add args you want: |like,so|
+    options["config_file"] = config_file
+    NewBackup::Main.new(options).run
+  end
+
+  # supplemental methods here
+
+  # Declare command-line interface here
+
+  description "Backup raw and clean data from RDS to S3"
+
+  # Accept flags via:
+  # on("--flag VAL","Some flag")
+  # options[flag] will contain VAL
+  #
+  # Specify switches via:
+  # on("--[no-]switch","Some switch")
+  on("--debug", "Turn on debug mode")
+  on("--nords", "Don't run the RDS portion")
+  on("--nos3",  "Don't run the S3 portion")
+  #
+  # Or, just call OptionParser methods on opts
+  #
+  # Require an argument
+  arg :config_file, "Specify the configuration file path to set options for the backup",
+  "See #{File.dirname(File.dirname(__FILE__))}/sample-config.yml as an example"
+  #
+  # # Make an argument optional
+  # arg :optional_arg, :optional
+
+  version NewBackup::VERSION
+
+  use_log_level_option
+
+  go!
+end

data/features/new_backup.feature

@@ -0,0 +1,14 @@
+Feature: My bootstrapped app kinda works
+  In order to get going on coding my awesome app
+  I want to have aruba and cucumber setup
+  So I don't have to do it myself
+
+  Scenario: App just runs
+    When I get help for "new_backup"
+    Then the exit status should be 0
+    And the banner should be present
+    And the banner should document that this app takes options
+    And the following options should be documented:
+      |--version|
+    And the banner should document that this app's arguments are:
+      |config_file|which is required|

data/features/step_definitions/new_backup_steps.rb

@@ -0,0 +1 @@
+# Put your step definitions here

data/features/support/env.rb

@@ -0,0 +1,16 @@
+require 'aruba/cucumber'
+require 'methadone/cucumber'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end

data/lib/new_backup.rb

@@ -0,0 +1,10 @@
+require "new_backup/version"
+require 'new_backup/datadog.rb'
+require 'new_backup/main.rb'
+require 'new_backup/myrds.rb'
+require 'new_backup/mys3.rb'
+require 'new_backup/mysqlcmds.rb'
+
+module NewBackup
+  # Your code goes here...
+end

data/lib/new_backup/datadog.rb

@@ -0,0 +1,92 @@
+=begin
+
+= datadog.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-01
+
+== Description
+
+Manage the connection and communication from the app to DataDog service.
+
+== Usage
+
+require '.../datadog.rb'
+
+class MyClass
+
+  include DataDog
+  api_key = 'Your DD API Key'
+  dogger "Hi from MyClass!"
+
+  begin
+    # some operations
+  rescue Exception => e
+    dogger "Woops, an Error", :type => :error, :body => "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+  end
+
+  # ... other class code
+
+end
+
+== Caveats
+
+This module integrates with methadone, using it's logging features in
+addition to sending the log message to data dog.  This isn't
+necessarily the most robust intermingling...
+
+=end
+
+require 'dogapi'
+require 'methadone'
+
+module NewBackup
+
+  module DataDog
+
+    include Methadone::CLILogging
+
+    @api_key = nil
+    @environment = (ENV['RAILS_ENV'].nil? || ENV['RAILS_ENV'].empty?) ? 'development' : ENV['RAILS_ENV']
+
+    module_function
+
+    def api_key
+      @api_key
+    end
+    
+    def api_key=(api_key)
+      @api_key=api_key
+    end
+
+    def dogger(msg, options={})
+
+      return if msg.nil? || msg.empty?
+
+      if options[:type] == :error
+        error msg
+      else
+        info msg
+      end
+      
+      return if @api_key.nil? || @api_key.empty?
+      
+      @client ||= Datadog::Client.new(@api_key)
+
+      emit_options = {
+        :msg_title => msg,
+        :alert_type => (options[:type] == :error) ? 'Error' : 'Normal',
+        :tags => [ "host:#{`hostname`}", "env:#{@environment}" ],
+        :priority => 'normal',
+        :source => 'MyApps'}
+
+      body = options[:body] ||= msg
+
+      @client.emit_event(Dogapi::Event.new(body, emit_options))
+
+    end
+
+  end
+
+end