rds-s3-backup 0.0.6 → 0.0.8

data/.rspec

@@ -1,2 +1,2 @@
 --color
---format progress
+--format documentation

data/Rakefile

@@ -1 +1,9 @@
 require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new do |t|
+  # spec options go in .rspec in root
+end
+
+
+task :default => [:spec]

data/bin/.gitignore

@@ -0,0 +1 @@
+/#rds-s3-backup.rb#

data/lib/rds-s3-backup.rb

@@ -14,18 +14,18 @@ module Rds
 
         options = thor_defaults
 
-        if thor_options[:config_file] && File.exists?(thor_options[:config_file])
+        if thor_options["config_file"] && File.exists?(thor_options["config_file"])
           begin
-            options = options.merge(YAML.load(File.read(thor_options[:config_file])))
+            options = options.merge(YAML.load(File.read(thor_options["config_file"])))
           rescue Exception => e
-            raise "Unable to read and parse #{thor_options[:config_file]}: #{e.class}: #{e}"
+            raise "Unable to read and parse #{thor_options["config_file"]}: #{e.class}: #{e}"
           end
         end
 
         options.merge!(thor_options)
 
         # Check for required options
-        missing_options = %w{rds_instance_id s3_bucket aws_access_key_id aws_secret_access_key mysql_database mysql_username mysql_password}.reduce([]) {|a, o| a << o unless options.has_key?(o); a}
+        missing_options = %w{rds_instance_id s3_bucket aws_access_key_id aws_secret_access_key mysql_database mysql_username mysql_password}.select {|o| o unless options.has_key?(o)}
 
         raise "Missing required options #{missing_options.inspect} in either configuration or command line" if missing_options.count > 0
         
@@ -36,8 +36,10 @@ module Rds
 
       def run(thor_options,thor_defaults)
 
-        @options = process_options(thor_options,thor_defaults)
         $logger = Logger.new(STDOUT)
+        @options = process_options(thor_options,thor_defaults)
+        $logger.debug "Running with Options: #{@options.to_yaml}"
+
         $logger.level = set_logger_level(@options["log_level"])
 
         $dogger = DataDog.new(@options['data_dog_api_key'])
@@ -50,24 +52,31 @@ module Rds
 
         $logger.debug "#{File.basename(__FILE__)}:#{__LINE__}: Running with options:"
         debug_opts = @options.dup
-        debug_opts['aws_access_key_id'] = 'X'*10
-        debug_opts['aws_secret_access_key'] = 'Y'*15
-        debug_opts['mysql_password'] = "ZZY"*5
-        debug_opts['data_dog_api_key'] = 'XYZZY'*3
-        $logger.debug debug_opts.to_yaml
+
+        $logger.debug @options.merge(
+        'aws_access_key_id' => 'X'*10,
+        'aws_secret_access_key' => 'Y'*15,
+        'mysql_password' => "ZZY"*5,
+        'data_dog_api_key' => 'XYZZY'*3                                     
+                                     ).to_yaml
 
         begin
 
           $logger.info "Creating RDS and S3 Connections"
           rds = MyRDS.new(@options)
-          s3 = MyS3.new(@options)
+          $logger.debug "rds: #{rds.to_yaml}"
 
-          $logger.info "Restoring Database"
+          $logger.info "Restoring Database"  
           rds.restore_db()
 
           $logger.info "Dumping and saving original database contents"
-          real_data_file = "#{rds.server.id}-mysqldump-#{@options['timestamp']}.sql.gz"
-          s3.save_production(rds.dump(real_data_file))
+          real_data_file = "#{rds.server.id}-mysqldump-#{@options['timestamp']}.sql.gz".
+            tap{|t| $logger.debug "real_data_file: #{t}"}
+ 
+          real_dump_file = rds.dump(real_data_file).tap{|t| $logger.debug "rds.dump returns: #{t}"}
+                    s3 = MyS3.new(@options)
+          $logger.debug "s3: #{s3.to_yaml}"
+          s3.save_production(real_dump_file)
 
           if @options['dump_ttl'] > 0
             $logger.info "Pruning old dumps"

data/lib/rds-s3-backup/myrds.rb

@@ -20,23 +20,26 @@ module Rds::S3::Backup
   class MyRDSException < RuntimeError ; end
   class MyRDS
 
-    attr_accessor :rds, :server
-
     def initialize(opts)
       @opts = opts
+    end
 
-      @rds = get_rds_connection(:aws_access_key_id => @opts['aws_access_key_id'],
-                                :aws_secret_access_key => @opts['aws_secret_access_key'],
-                                :region => @opts['aws_region'])
-
-      @server = get_rds_server(@opts['rds_instance_id'])
+    def rds
+      # Memoize @rds
+      @rds ||= get_rds_connection()
+    end
 
+    def server
+      # Memoize @server
+      @server ||= get_rds_server(@opts['rds_instance_id'])
     end
 
+
+    # Restore the production database from the most recent snapshot
     def restore_db
 
       begin
-        @rds.restore_db_instance_from_db_snapshot(new_snap.id,
+        self.rds.restore_db_instance_from_db_snapshot(new_snap.id,
                                                   backup_server_id,
                                                   {"DBSubnetGroupName" => @opts['db_subnet_group_name'],
                                                     "DBInstanceClass" => @opts['db_instance_type'] } )
@@ -47,6 +50,7 @@ module Rds::S3::Backup
     end
 
 
+    # Dump the database to the backup file name
     def dump(backup_file_name)
       @mysqlcmds ||= ::Rds::S3::Backup::MySqlCmds.new(backup_server.endpoint['Address'],
                                                       @opts['mysql_username'],
@@ -59,15 +63,13 @@ module Rds::S3::Backup
       @mysqlcmds.dump(backup_file_path(backup_file_name)) # returns the dump file path
     end
 
+    # Convert personal data in the production data into random generic data 
     def obfuscate
       @mysqlcmds ||= ::Rds::S3::Backup::MySqlCmds.new(backup_server.endpoint['Address'],
                                                       @opts['mysql_username'],
                                                       @opts['mysql_password'],
                                                       @opts['mysql_database'])
 
-
-
-
       @mysqlcmds.exec(@opts['obfuscate_sql'])
     end
 
@@ -76,7 +78,7 @@ module Rds::S3::Backup
     end
 
     def backup_server_id
-      @backup_server_id ||= "#{@server.id}-s3-dump-server-#{@opts['timestamp']}"
+      @backup_server_id ||= "#{self.server.id}-s3-dump-server-#{@opts['timestamp']}"
     end
 
     def new_snap
@@ -103,11 +105,11 @@ module Rds::S3::Backup
       @snap_name ||= "s3-dump-snap-#{@opts['timestamp']}"
     end
 
-    def get_rds_connection(opts={})
+    def get_rds_connection()
       options = {
         :aws_access_key_id => @opts['aws_access_key_id'],
         :aws_secret_access_key => @opts['aws_secret_access_key'],
-        :region => @opts['aws_region']}.merge(opts)
+        :region => @opts['aws_region']}
       Fog.timeout=@opts['fog_timeout']
       begin
         connection = Fog::AWS::RDS.new(options)
@@ -116,14 +118,15 @@ module Rds::S3::Backup
       end
       
       raise MyRDSException.new("Unable to make RDS connection") if connection.nil?
-      connection
 
+      connection
 
     end
 
     def get_rds_server(id)
-      begin
-        server = @rds.servers.get(id)
+
+       begin
+        server = self.rds.servers.get(id)
       rescue Exception => e
         raise MyRDSException.new("Error getting server in #{self.class}#get_rds_server: #{e.class}: #{e}")
       end

data/lib/rds-s3-backup/mys3.rb

@@ -18,12 +18,79 @@ module Rds::S3::Backup
 
     def initialize(options)
       @options = options
+    end
+
+    # Lazy loaders
+    def s3
+      @s3 ||= get_storage(:aws_access_key_id => @options['aws_access_key_id'],
+                          :aws_secret_access_key => @options['aws_secret_access_key'],
+                          :region => @options['aws_s3_region'] ||= @options['aws_region'])
+    end
 
-      @s3 = get_storage(:aws_access_key_id => @options['aws_access_key_id'],
-                        :aws_secret_access_key => @options['aws_secret_access_key'],
-                        :region => @options['aws_s3_region'] ||= @options['aws_region'])
+    def s3_bucket
+      @s3_bucket ||= get_bucket(@options['s3_bucket'])
     end
 
+    def backup_bucket
+      @backup_bucket ||= get_bucket(@options['backup_bucket'])
+    end
+
+
+
+    # Save the Real version of the Production Database
+    #
+    # == Input
+    #
+    # :file_path:: the path where the production database resides to save up to S3
+    #
+    def save_production(file_path)
+      save(s3_bucket, file_path, :acl => 'private')
+    end
+
+    
+    # Save the Cleaned (Obfuscated) Version of the Database
+    #
+    # == Input
+    #
+    # :file_path:: the path where the file resides to save up to S3
+    #
+    def save_clean(file_path)
+      save(backup_bucket, file_path, :acl => 'authenticated-read')
+    end
+
+
+    # Remove older files from S3
+    #
+    # == Input
+    #
+    # o -- an options hash, expecting the following keys:
+    #
+    # :prefix:: the prefix to use with the s3 bucket
+    # :keep::   the number of files to keep. Must keep at least one.
+    def prune_files(o={})
+      options = {
+        :prefix => '',
+        :keep => 1
+      }.merge(o)
+      
+      raise MyS3Exception "Must keep at least one file. options[:keep] = #{options[:keep]}" if options[:keep] < 1 # must keep at least one, the last one!
+
+      my_files = s3_bucket.files.all('prefix' => options[:prefix])
+      return if my_files.nil?
+      
+      if my_files.count > options[:keep]
+        my_files.
+          sort {|x,y| x.last_modified <=> y.last_modified}.
+          take(files_by_date.count - options[:keep]).
+          each do |f|
+          $logger.info "Deleting #{f.name}"
+          f.destroy
+        end
+      end
+    end
+
+
+    # Make a connection to AWS S3 using Fog::Storage
     def get_storage(o={})
       options = {
         :aws_access_key_id => nil,
@@ -38,33 +105,30 @@ module Rds::S3::Backup
         raise MyS3Exception.new "Error establishing storage connection: #{e.class}: #{e}"
       end
       
+      $logger.debug "What is storage? #{storage.class}:#{storage.inspect}"
       raise MyS3Exception.new "In #{self.class}#get_storage: storage is nil!" if storage.nil?
 
+      storage
     end
 
+
+    # Retrieve a pointer to an AWS S3 Bucket
     def get_bucket(bucket)
 
       begin
-        bucket = @s3.directories.get(bucket)
+        bucket = self.s3.directories.get(bucket)
       rescue Exception => e
         raise MyS3Exception.new "Error getting bucket #{bucket} in S3: #{e.class}: #{e}"
       end
 
       raise MyS3Exception.new "In #{self.class}#get_bucket: bucket is nil!" if bucket.nil?
 
+      bucket
     end
 
-    def save_production(file_path)
-      save(s3_bucket, file_path, :acl => 'private')
-    end
-
-    def save_clean(file_path)
-      save(backup_bucket, file_path, :acl => 'authenticated-read')
-    end
-
-
-
+    # Perform the actual save from a local file_path to AWS S3
     def save(bucket, file_path, o={})
+      raise MyS3Exception.new "bucket is nil!!" if bucket.nil?
       options = {
         :key => File.join(@options['s3_prefix'], File.basename(file_path)),
         :body => File.open(file_path),
@@ -79,8 +143,9 @@ module Rds::S3::Backup
 
       rescue Exception => e
         if tries < 3
-          @logger.info "Retrying S3 upload after #{tries} tries"
+          $logger.info "Retrying S3 upload after #{tries} tries"
           tries += 1
+          sleep tries * 60      # progressive back off
           retry
         else
           raise MyS3Exception.new "Could not save #{File.basename(file_path)} to S3 after 3 tries: #{e.class}: #{e}"
@@ -89,40 +154,8 @@ module Rds::S3::Backup
 
     end
 
-
-    def s3_bucket
-      @s3_bucket ||= get_bucket(@options['s3_bucket'])
-    end
-
-    def backup_bucket
-      @backup_bucket ||= get_bucket(@options['backup_bucket'])
-    end
-
-
-    def prune_files(o={})
-      options = {
-        :prefix => '',
-        :keep => 1
-      }.merge(o)
-      
-      return if options[:keep] < 1 # must keep at least one, the last one!
-
-      my_files = s3_bucket.files.all('prefix' => options[:prefix])
-      return if my_files.nil?
-      
-      if my_files.count > options[:keep]
-        my_files.
-          sort {|x,y| x.last_modified <=> y.last_modified}.
-          take(files_by_date.count - options[:keep]).
-          each do |f|
-          logger.info "Deleting #{f.name}"
-          f.destroy
-        end
-      end
-    end
-
+    # nothing really to do here...    
     def destroy
-      # nothing really to do here...
     end
 
 

data/lib/rds-s3-backup/version.rb

@@ -22,7 +22,7 @@ module Rds
       #  2013-03-04 tamara
 
 
-      VERSION = "0.0.6"
+      VERSION = "0.0.8"
     end
   end
 end

data/spec/rds-s3-backup/datadog_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+require 'dogapi'
+require 'rds-s3-backup'
+
+module Rds::S3::Backup
+
+  describe "DataDog" do
+    it { DataDog.should respond_to(:new) }
+
+    describe "#make_dog_client" do
+
+      it "should return the client object" do
+        Dogapi::Client.stub(:new).with("apikey").and_return("apikey")
+        DataDog.new("apikey").make_dog_client.should == "apikey"
+      end
+
+      it "should raise an exception" do
+        Dogapi::Client.stub(:new) { raise RuntimeError }
+        expect { DataDog.new("apikey").make_dog_client }.to raise_error(RuntimeError)
+      end
+
+    end
+
+    describe "#send" do
+      it "should do something interesting" do
+        Dogapi::Client.stub_chain(:new, :emit_event) {"event"}
+        Dogapi::Event.stub(:new)
+        DataDog.new("apikey").send("message").should == "event"
+      end
+
+      it "should raise and exception" do
+        Dogapi::Client.stub_chain(:new, :emit_event) { raise RuntimeError }
+        expect { DataDog.new("apikey").send("message")}.to raise_error(RuntimeError)
+      end
+
+
+
+    end
+
+
+
+
+  end
+
+end

data/spec/rds-s3-backup/myrds_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+require 'rds-s3-backup'
+require 'fog'
+
+module Rds::S3::Backup
+
+  describe "MyRDS" do
+
+    let(:myrds_options) {{'aws_access_key_id' => 'abcdef',
+      'aws_secret_access_key' => 'xyzzy',
+      'region' => 'us-west',
+      'rds_instance_id' => 'honeybadger',
+      'db_subnet_group_name' => 'test',
+      'db_instance_type' => 'big',
+      'mysql_username' => 'JoeBob',
+      'mysql_password' => 'BillyBriggs',
+      'mysql_database' => 'test',
+      'dump_directory' => '.',
+      'obfuscate_sql' => 'obfuscate_database.sql',
+      'timestamp' => Time.new.strftime('%Y-%m-%d-%H-%M-%S-%Z')
+    }}
+      
+    let(:myrds) {MyRDS.new(myrds_options)}
+
+    before(:each) do
+      Fog.stub(:timeout=)
+      Fog::AWS::RDS.stub(:new) { Object.new }
+    end
+    
+    it ":get_rds_connection should return an exception" do
+      Fog::AWS::RDS.stub(:new) { raise RuntimeError }
+      expect { myrds.get_rds_connection }.to raise_error(RuntimeError)
+    end
+
+    xit "execute :dump" do
+      Fog::AWS::RDS.stub_chain(:new, :servers)
+      Rds::S3::Backup::MySqlCmds.stub_chain(:new, :dump) {"dumpfile.sql.gz"}
+      myrds.dump("dumpfile.sql.gz").should == "dumpfile.sql.gz"
+    end
+
+    xit "execute :obfuscate" do
+      Rds::S3::Backup::MySqlCmds.stub_chain(:new, :exec)
+      myrds.exec(myrds_options['obfuscate_sql']).should == 0
+    end
+
+    xit "executing :obfuscate returns an error" do
+      Rds::S3::Backup::MySqlCmds.stub_chain(:new, :exec) {raise RuntimeError}
+      expect {myrds.exec(myrds_options['obfuscate_sql'])}.to raise_error(RuntimeError)
+    end
+
+    
+  end
+
+
+end

data/spec/rds-s3-backup_spec.rb

@@ -17,10 +17,6 @@ require 'rds-s3-backup'
 
 module Rds::S3::Backup
 
-  describe "has version" do
-    it {Rds::S3::Backup::VERSION.should == '0.0.1' }
-  end
-  
   describe "module methods" do
     
     it { Rds::S3::Backup.should respond_to(:run) }
@@ -31,54 +27,55 @@ module Rds::S3::Backup
 
   describe "#run method" do
       let(:options) { {
-          :rds_instance_id => 'stagingdb',
-          :s3_bucket => 'novu-backups',
-          :backup_bucket => 'novu-backups',
-          :s3_prefix => 'db_dumps',
-          :aws_access_key_id => 'ABCDE',
-          :aws_secret_access_key => '0987654321',
-          :mysql_database => 'novu_text',
-          :mysql_username => 'novurun',
-          :mysql_password => 'passw0rd',
-          :obfuscate_sql => '/usr/local/etc/obfuscate.sql',
-          :dump_ttl => 0,
-          :dump_directory => '/mnt/secure',
-          :aws_region => 'us-east-1',
-          :aws_s3_region => 'us-west-2',
-          :db_subnet_group_name => 'staging db subnet',
-          :db_instance_type => 'db.m1.small',
-          :instance_id => '',
-          :log_level => 0,
-          :quiet => false,
-          :verbose => true
+          "rds_instance_id" => 'stagingdb',
+          "s3_bucket" => 'novu-backups',
+          "backup_bucket" => 'novu-backups',
+          "s3_prefix" => 'db_dumps',
+          "aws_access_key_id" => 'ABCDE',
+          "aws_secret_access_key" => '0987654321',
+          "mysql_database" => 'novu_text',
+          "mysql_username" => 'novurun',
+          "mysql_password" => 'passw0rd',
+          "obfuscate_sql" => '/usr/local/etc/obfuscate.sql',
+          "dump_ttl" => 0,
+          "dump_directory" => '/mnt/secure',
+          "aws_region" => 'us-east-1',
+          "aws_s3_region" => 'us-west-2',
+          "db_subnet_group_name" => 'staging db subnet',
+          "db_instance_type" => 'db.m1.small',
+          "instance_id" => '',
+          "log_level" => 'debug',
+          "quiet" => false,
+          "verbose" => true
         } }
 
 
     before(:each) do
-      DataDog.stub(:new).and_return(OpenStruct.new(:send => true))
-
-      MyRDS.stub(:new).and_return(OpenStruct.new(:restore_db => true,
-                                                 :dump => Proc.new {|file| file },
-                                                 :obfuscate => true,
-                                                 :destroy => true,
-                                                 :server =>
-                                                 OpenStruct.new(:id => 1)
-                                                 ))
-      MyS3.stub(:new).and_return(OpenStruct.new(:save_production => true,
-                                                :save_clean => true,
-                                                :prune_files => true,
-                                                :destroy => true))
+      DataDog.stub_chain(:new, :send)
+
+
+      MyRDS.stub_chain(:new, :restore_db)
+      MyRDS.stub_chain(:new, :destroy)
+      MyRDS.stub_chain(:new, :server, :id).and_return 1
+      MyRDS.stub_chain(:new, :obfuscate)
+      MyRDS.stub_chain(:new, :dump).and_return "dump_path"
+
+      MyS3.stub_chain(:new, :save_production)
+      MyS3.stub_chain(:new, :save_clean)
+      MyS3.stub_chain(:new, :prune_files)
+      MyS3.stub_chain(:new, :destroy)
+
     end
 
     it "should process options" do
-      opts = Rds::S3::Backup.process_options(options)
+      opts = Rds::S3::Backup.process_options(options,{})
       opts.should be_a(Hash)
-      opts.should have_key(:timestamp)
-      opts[:timestamp].should_not be_nil
+      opts.should have_key("timestamp")
+      opts["timestamp"].should_not be_nil
     end
 
 
-    it { Rds::S3::Backup.run(options).should == 0 }
+    it { Rds::S3::Backup.run(options,{}).should == 0 }
 
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rds-s3-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-07 00:00:00.000000000 Z
+date: 2013-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -160,6 +160,7 @@ description: ! '"Thor script and libraries to backup an AWS RDS snapshot to AWS
 email:
 - tamouse@gmail.com
 executables:
+- .gitignore
 - rds-s3-backup.rb
 extensions: []
 extra_rdoc_files: []
@@ -171,6 +172,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/.gitignore
 - bin/rds-s3-backup.rb
 - lib/rds-s3-backup.rb
 - lib/rds-s3-backup/datadog.rb
@@ -179,6 +181,8 @@ files:
 - lib/rds-s3-backup/mysqlcmds.rb
 - lib/rds-s3-backup/version.rb
 - rds-s3-backup.gemspec
+- spec/rds-s3-backup/datadog_spec.rb
+- spec/rds-s3-backup/myrds_spec.rb
 - spec/rds-s3-backup_spec.rb
 - spec/spec_helper.rb
 - test_data/.gitignore
@@ -209,6 +213,8 @@ signing_key:
 specification_version: 3
 summary: ! '"Backup from AWS RDS snapshot to AWS S3 as mysqldump"'
 test_files:
+- spec/rds-s3-backup/datadog_spec.rb
+- spec/rds-s3-backup/myrds_spec.rb
 - spec/rds-s3-backup_spec.rb
 - spec/spec_helper.rb
 has_rdoc: