new_backup 1.0.1

data/lib/new_backup/mysqlcmds.rb

@@ -0,0 +1,95 @@
+=begin
+
+= mysqlcmds.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-01
+*License*::      MIT
+*Version*::      0.0.1
+
+== Description
+
+Class to wrap mysql command line applications mysqldump, mysqladmin, and mysql.
+
+=end
+
+require 'methadone'
+require 'RunIt'
+
+module NewBackup
+
+  class MySqlCmds
+
+    include Methadone::CLILogging
+
+    attr_accessor :hostname, :username, :password, :database, :obfuscate_script
+
+
+    def initialize(hostname, username, password, database, obfuscate)
+      self.hostname=hostname.dup.to_s
+      self.username=username.dup.to_s
+      self.password=password.dup.to_s
+      self.database=database.dup.to_s
+      self.obfuscate_script=obfuscate.dup.to_s
+
+      @commands =
+        {:mysqldump  	=> get_command('mysqldump'),
+        :mysql      	=> get_command('mysql'),
+        :gzip		=> get_command('gzip')
+      }
+
+      debug "#{self.class}##{__method__}:#{__LINE__}: @commands: #{@commands}"
+
+    end
+
+    def dump(save_file)
+      cmd = []
+      cmd << @commands[:mysqldump]
+      cmd << "--host #{self.hostname}"
+      cmd << "--user=#{self.username}"
+      cmd << "--password=#{self.password}" unless self.password.nil? or self.password.empty?
+      cmd << self.database
+      cmd << "|"
+      cmd << @commands[:gzip]
+      cmd << ">"
+      cmd << save_file
+ 
+      debug "#{self.class}##{__method__}:#{__LINE__}: cmd = #{cmd.join(" ")}"
+
+      saver = RunIt.new cmd.join(" ")
+      saver.run
+      raise "#{self.class}#save error: #{saver.result.exitstatus}: #{saver.error.inspect}" unless saver.result.success?
+      debug "#{self.class}##{__method__}:#{__LINE__}: saver.output: #{saver.output}"
+    end
+
+    def obfuscate
+
+      cmd = []
+      cmd << @commands[:mysql]
+      cmd << "--host #{self.hostname}"
+      cmd << "--user #{self.username}"
+      cmd << "--password=#{self.password}" unless self.password.nil? or self.password.empty?
+      cmd << self.database
+      cmd << "<"
+      cmd << self.obfuscate_script
+
+      debug "#{self.class}##{__method__}:#{__LINE__}: cmd= #{cmd.join(" ")}"
+      obfuscator = RunIt.new cmd.join(" ")
+      obfuscator.run
+      raise "#{self.class}##{__method__}:#{__LINE__}: error: #{obfuscator.result.exitstatus}: #{obfuscator.error.inspect}" unless obfuscator.result.success?
+      debug "#{self.class}##{__method__}:#{__LINE__}: obfuscator.output: #{obfuscator.output}"
+    end
+
+
+    private
+
+    def get_command(command)
+      command_path = `which #{command}`.chomp
+      raise "#{command} not found!" if command_path.empty? || command_path.nil?
+      command_path
+    end
+
+  end
+
+end

data/lib/new_backup/version.rb

@@ -0,0 +1,17 @@
+=begin
+
+= new_backup/version.rb
+
+*Copyright*::     (C) 2013 by Novu, LLC
+*Author(s)*::     Tamara Temple <tamara.temple@novu.com>
+
+=end
+
+
+
+
+module NewBackup
+  VERSION = "1.0.1"
+  DESCRIPTION = "Backup an RDS database instance to S3, and create and obfuscated version"
+  SUMMARY = DESCRIPTION
+end

data/new_backup.gemspec

@@ -0,0 +1,27 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'new_backup/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "new_backup"
+  gem.version       = NewBackup::VERSION
+  gem.authors       = ["Tamara Temple"]
+  gem.email         = ["tamouse@gmail.com"]
+  gem.description   = NewBackup::DESCRIPTION
+  gem.summary       = NewBackup::SUMMARY
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_development_dependency('rdoc')
+  gem.add_development_dependency('aruba')
+  gem.add_development_dependency('rake', '~> 0.9.2')
+  gem.add_dependency('methadone', '~> 1.2.6')
+  gem.add_dependency('RunIt')
+  gem.add_dependency('fog')
+  gem.add_dependency('dogapi')
+  gem.add_development_dependency('rspec')
+  gem.add_development_dependency('debugger')
+end

data/sample-config.yml

@@ -0,0 +1,20 @@
+# Sample Config file for the RDS->S3 backup script
+fog_timeout: 3600
+dump_directory: /tmp
+dump_ttl: 20
+aws_region: us-west-1
+db_instance_type: db.t1.micro
+timestamp_format: "%FT%T-%X"
+rds_instance_id: myrds
+db_subnet_group_name: yolo
+backup_bucket: raw_backups
+s3_bucket: clean_backups
+s3_prefix: felix
+aws_s3_region: outermars
+mysql_username: despereaux
+mysql_database: gollywogs
+mysql_password: nutz
+obfuscate_script: nolocontendre
+datadog_apikey: youdthink
+aws_access_key_id: bogus aws key
+aws_secret_access_key: bogus aws secret

data/spec/datadog_spec.rb

@@ -0,0 +1,31 @@
+=begin
+
+= datadog_spec.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-02
+*License*::      GPLv3
+*Version*::      0.0.1
+
+=end
+
+require 'spec_helper'
+require 'new_backup/datadog'
+
+
+module NewBackup
+
+  describe DataDog do
+    
+    it "#api_key=" do
+      NewBackup::DataDog.api_key= '12345'
+      NewBackup::DataDog.api_key.should == '12345'
+    end
+
+
+
+  end
+
+
+end

data/spec/main_spec.rb

@@ -0,0 +1,77 @@
+=begin
+
+= new_backup_spec.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-01
+*License*::      GPLv3
+*Version*::      0.0.1
+
+== Description
+
+Test base library module.
+
+=end
+
+require 'spec_helper'
+require 'new_backup/main'
+
+def check_keys(h,klist)
+  (h.keys <=> klist) == 0
+end
+
+module NewBackup
+
+  describe Main do
+
+    describe "processing options and initializing" do
+      let(:dogger) {double('dogger')}
+
+      it "no options specified" do
+        expect {NewBackup::Main.new()}.to raise_error(RuntimeError)
+      end
+
+      describe "full options" do
+        let(:options) {{
+            'fog_timeout' => 3600,
+            'dump_directory' => '/tmp',
+            'dump_ttl' => 20,
+            'aws_region' => 'us-west-1',
+            'db_instance_type' => 'db.t1.micro',
+            'timestamp_format' => "%FT%T-%Z",
+            'rds_instance_id' => 'myrds',
+            'db_subnet_group_name' => 'yolo',
+            'backup_bucket' => 'raw_backups',
+            's3_bucket' => 'clean_backups',
+            's3_prefix' => 'felix',
+            'aws_s3_region' => 'outermars',
+            'mysql_username' => 'despereaux',
+            'mysql_database' => 'gollywogs',
+            'mysql_password' => 'nutz',
+            'obfuscate_script' => 'nolocontendre',
+            'datadog_apikey' => 'youdthink',
+            'aws_access_key_id' => 'bogus aws key',
+            'aws_secret_access_key' => 'bogus aws secret'
+          }}
+        let(:main) { NewBackup::Main.new(options) }
+        let(:actual_options) { main.options }
+
+        it {main.should be_a(NewBackup::Main)}
+        it {check_keys(actual_options, %w{fog aws rds s3 mysql datadog dump_directory timestamp debug nords nos3}.map(&:to_sym)).should be_true}
+        it {check_keys(actual_options[:fog], %w{timeout}.map(&:to_sym)).should be_true}
+        it {check_keys(actual_options[:aws], %w{access_key secret_key region}.map(&:to_sym)).should be_true}
+        it {check_keys(actual_options[:rds], %w{instance_id subnet_group instance_type}.map(&:to_sym)).should be_true}
+        it {check_keys(actual_options[:s3], %w{raw_bucket clean_bucket prefix region dump_ttl}.map(&:to_sym)).should be_true}
+        it {check_keys(actual_options[:mysql], %w{username password database obfuscate_script}.map(&:to_sym)).should be_true}
+        it {check_keys(actual_options[:datadog], %w{api_key}.map(&:to_sym)).should be_true}
+
+        it { main.should respond_to(:run) }
+        
+      end
+
+    end
+
+  end
+
+end

data/spec/mockbucket.rb

@@ -0,0 +1,83 @@
+=begin
+
+= mockbucket.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-14
+*License*::      GPLv3
+*Version*::      0.0.1
+
+== Description
+
+=end
+
+class MockFile
+  attr_accessor :name, :last_modified, :key, :directory
+  def initialize(n,d,dir)
+    self.name = n.dup
+    self.key = self.name
+    self.last_modified = d.dup
+    self.directory = dir
+  end
+
+  def destroy
+    self.name = nil
+    self.key = nil
+    self.last_modified = nil
+  end
+end
+
+class MockBucket
+  attr_accessor :bucket_contents, :name, :key
+  def initialize(n=1, name)
+    self.bucket_contents = (1..n).to_a.map{|i| MockFile.new("file-#{i}", Time.new(2012, 6, (i+1), 0, 0, 0), self) }
+    self.name = name.dup
+    self.key = self.name
+  end
+  
+  def files
+    self
+  end
+
+  def create(options={})
+    self.bucket_contents << MockFile.new(options[:key], Time.now, self)
+  end
+
+  def all(options={})
+    @bucket_contents.reject {|f| f.name.nil?}
+  end
+
+end
+
+
+class MockConnection
+  attr_accessor :bucket_list, :name, :key
+  def initialize(n=1, name)
+    self.name = name.dup
+    self.key = self.name
+    if n > 0
+      self.bucket_list = (1..n).to_a.map{|i| MockBucket.new(10, "bucket-#{i}")}
+    else
+      self.bucket_list = []
+    end
+
+  end
+
+  def directories
+    @bucket_list
+  end
+
+  def get(name)
+    @bucket_list.select {|bucket| bucket.name == name}.first
+  end
+
+  def create(name)
+    @bucket_list << MockBucket.new(10, name)
+  end
+
+  def destroy(name)
+    @bucket_list.reject! {|bucket| bucket.name == name}
+  end
+
+end

data/spec/myrds_spec.rb

@@ -0,0 +1,131 @@
+=begin
+
+= myrds_spec.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-01
+*License*::      GPLv3
+*Version*::      0.0.1
+
+== Description
+
+Test myrds.rb
+
+=end
+
+require 'spec_helper'
+require 'new_backup/myrds.rb'
+require 'fog'
+require 'date'
+
+module NewBackup
+
+  describe MyRds do
+
+    it {NewBackup::MyRds.should respond_to(:new)}
+
+    describe "Operations" do
+
+
+      it "#connect" do
+        Fog.mock!
+        options = {
+          :aws => {
+            :access_key => 'bogus access key',
+            :secret_key => 'bogus secret key',
+            :rds_region => 'us-east-1'},
+          :fog => {
+            :timeout => 10
+          },
+          :timestamp => DateTime.now.iso8601
+        }
+
+        NewBackup::MyRds.new(options).connect do |connection|
+          connection.should_not be_nil
+        end
+      end
+
+      it "#retrieve_snapshot" do
+        
+        Fog.mock!
+        options = {
+          :rds => {
+            :instance_id => 'tamtest'
+          },
+          :aws => {
+            :access_key => 'bogus access key',
+            :secret_key => 'bogus secret key',
+            :rds_region => 'us-east-1'},
+          :fog => {
+            :timeout => 10},
+          :timestamp => DateTime.now.iso8601
+        }
+
+        rds_server = double('rds_server')
+        snapshot = double('snapshot')
+        rds_server.stub_chain(:snapshots, :new, :save)
+        rds_server.stub_chain(:snapshots, :get) {snapshot}
+        snapshot.stub(:wait_for)
+        snapshot.stub(:destroy)
+
+        NewBackup::MyRds.new(options).retrieve_snapshot(rds_server) do |snapshot|
+          snapshot.should_not be_nil
+        end
+
+      end
+
+      it "#restore_db" do
+        
+        Fog.mock!
+        options = {
+          :rds => {
+            :instance_id => 'tamtest',
+            :subnet_group => '',
+            :instance_type => 'db.t1.micro'
+          },
+          :aws => {
+            :access_key => 'bogus access key',
+            :secret_key => 'bogus secret key',
+            :rds_region => 'us-east-1'},
+          :mysql => {
+            :database   => 'qadb',
+            :username   => 'root',
+            :password   => 'aaaa',
+            :obfuscate_script => 'zzzz'
+          },
+          :fog => {
+            :timeout => 10
+          },
+          :timestamp => DateTime.now.iso8601
+        }
+        connection = double('connection')
+        snapshot = double('snapshot')
+        backup_server = double('backup_server')
+        connection.stub(:restore_db_instance_from_db_snapshot)
+        connection.stub_chain(:servers, :get) {backup_server}
+        snapshot.stub(:id)
+        backup_server.stub_chain(:endpoint, :[]).with("Address").and_return("localhost")
+        backup_server.stub(:wait_for)
+        backup_server.stub(:destroy)
+
+        NewBackup::MyRds.new(options).restore_db(connection, snapshot) do |db|
+          db.should_not be_nil
+        end
+
+      end
+
+    end
+
+
+    describe "#snap_name" do
+      let(:timestamp) {DateTime.now.iso8601}
+      it {NewBackup::MyRds.new(:timestamp => timestamp).snap_name.should == "s3-dump-snap-#{timestamp}" }
+    end
+
+    
+
+
+  end
+
+end