new_backup 1.0.1

data/lib/new_backup/main.rb

@@ -0,0 +1,193 @@
+=begin
+
+= main.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-01
+*License*::      MIT
+*Version*::      0.0.1
+
+== Description
+
+Main class/routine for new_backup.
+
+=end
+
+require 'methadone'
+require 'new_backup/myrds'
+require 'new_backup/mys3'
+require 'new_backup/mysqlcmds'
+require 'new_backup/datadog'
+
+module NewBackup
+
+  class Main
+
+    REQUIRED_OPTIONS = %w{rds_instance_id s3_bucket aws_access_key_id aws_secret_access_key mysql_database mysql_username mysql_password timestamp}
+    DEFAULT_OPTIONS = {
+      'fog_timeout' => 1800,
+      'dump_directory' => '/tmp',
+      'dump_ttl' => 0,
+      'aws_region' => 'us-east-1',
+      'db_instance_type' => 'db.m1.small',
+      'timestamp_format' => '%Y-%m-%d-%H-%M-%S-%Z'
+    }
+
+    include Methadone::CLILogging
+    include NewBackup::DataDog
+
+    attr_accessor :options
+
+    # Initialize the Main loop, processing options and putting them in the right form
+    def initialize(opts={})
+
+      debug "#{self.class}##{__method__}:#{__LINE__}: original opts: #{opts.to_yaml}"
+
+      if opts["config_file"] && File.exists?(opts["config_file"])
+        converged_opts = DEFAULT_OPTIONS.merge(YAML.load(File.read(opts["config_file"]))).merge(opts)
+      else
+        converged_opts = DEFAULT_OPTIONS.merge(opts)
+      end
+
+      converged_opts["timestamp"] = Time.now.strftime(converged_opts["timestamp_format"])
+      
+      debug "#{self.class}##{__method__}:#{__LINE__}: converged_opts: #{converged_opts.to_yaml}"
+
+      missing_options = REQUIRED_OPTIONS.select {|o| o unless converged_opts.has_key?(o)}
+      raise "Missing required options #{missing_options.inspect} in either configuration or command line" if missing_options.count > 0
+      
+      @options = {
+        :fog => {
+          :timeout      => converged_opts["fog_timeout"]
+        },
+        :aws => {
+          :access_key 	=> '[HIDDEN]',
+          :secret_key 	=> '[HIDDEN]',
+          :region 	=> converged_opts["aws_region"]
+        },
+        :rds => {
+          :instance_id	=> converged_opts["rds_instance_id"],
+          :subnet_group => converged_opts["db_subnet_group_name"],
+          :instance_type => converged_opts["db_instance_type"]
+        },
+        :s3 => {
+          :raw_bucket	=> converged_opts["s3_bucket"],
+          :clean_bucket => converged_opts["backup_bucket"],
+          :prefix	=> converged_opts["s3_prefix"],
+          :region	=> converged_opts["aws_s3_region"] ||= converged_opts["aws_region"],
+          :dump_ttl	=> converged_opts["dump_ttl"]
+        },
+        :mysql => {
+          :username	=> converged_opts["mysql_username"],
+          :password	=> '[HIDDEN]',
+          :database	=> converged_opts["mysql_database"],
+          :obfuscate_script => converged_opts["obfuscate_script"]
+        },
+        :datadog => {
+          :api_key	=> '[HIDDEN]'
+        },
+        :dump_directory => converged_opts["dump_directory"],
+        :timestamp	=> converged_opts["timestamp"],
+        :debug          => converged_opts["debug"],
+        :nords          => converged_opts["nords"],
+        :nos3           => converged_opts["nos3"]
+      }
+
+      
+      debug "Options:\n#{@options.to_yaml}"
+
+      # Fill in the hidden values after showing the options
+      @options[:aws][:access_key]   = converged_opts["aws_access_key_id"]
+      @options[:aws][:secret_key]   = converged_opts["aws_secret_access_key"]
+      @options[:mysql][:password]   = converged_opts["mysql_password"]
+      @options[:datadog][:api_key]  = converged_opts["datadog_apikey"]
+
+      debug @options.to_yaml
+
+
+    end
+
+    def run
+      dogger "Beginning RDS-S3-Backup"
+      begin
+        
+        raise "#{self.class}##{__method__}:#{__LINE__}: Dump directory #{@options[:dump_directory]} does not exist!" unless File.directory?(@options[:dump_directory])
+
+
+        raw_file = File.join(File.expand_path(@options[:dump_directory]),save_file_name)
+        debug "#{self.class}##{__method__}:#{__LINE__}: raw_file: #{raw_file}"
+        clean_file = File.join(File.expand_path(@options[:dump_directory]),clean_file_name)
+        debug "#{self.class}##{__method__}:#{__LINE__}: clean_file: #{clean_file}"
+
+        if (@options[:nords])
+          info "Not running RDS"
+          File.open(raw_file,'w') do |f|
+            f.puts "default content when not running RDS"
+          end
+          File.open(clean_file,'w') do |f|
+            f.puts "default content when not running RDS"
+          end
+        else
+
+          NewBackup::MyRds.new(@options).restore do |db|
+            info "Dump raw database"
+            db.dump(raw_file)
+            info "Obfuscate database"
+            db.obfuscate
+            info "Dump clean database"
+            db.dump(clean_file)
+          end
+        end
+
+        if (@options[:nos3])
+          info "Not running S3"
+        else
+          
+          s3 = NewBackup::MyS3.new(@options)
+          s3.connect do |connection|
+            s3.connect_bucket(connection, @options[:s3][:raw_bucket]) do |bucket|
+              info "Save raw db dump"
+              s3.put_file bucket, raw_file
+              info "Prune excess backups"
+              s3.prune bucket, @options[:s3][:dump_ttl]
+            end
+            
+            s3.connect_bucket(connection, @options[:s3][:clean_bucket]) do |bucket|
+              info "Save cleaned db dump"
+              s3.put_file bucket, clean_file
+            end
+          end
+          
+        end
+
+      rescue Exception => e
+        dogger "Fatal error in #{self.class}#run: #{e.class}: #{e}" ,
+        :type => :error,
+        :body => "Backtrace:\n#{e.backtrace.join("\n")}"
+        debug e.backtrace.join("\n")
+        raise e
+
+
+      ensure
+        File.unlink(raw_file) if File.exists?(raw_file) && ! @options[:debug]
+        File.unlink(clean_file) if File.exists?(clean_file) && ! @options[:debug]
+      end
+
+      dogger "End RDS-S3-Backup"
+
+    end
+
+    private
+
+    def save_file_name
+      "#{@options[:rds][:instance_id]}-mysqldump-#{@options[:timestamp]}.sql.gz"
+    end
+
+    def clean_file_name
+      "clean-mysqldump.sql.gz"
+    end
+
+  end
+
+end

data/lib/new_backup/myrds.rb

@@ -0,0 +1,140 @@
+=begin
+
+= myrds.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-01
+*License*::      MIT
+*Version*::      0.0.1
+
+== Description
+
+Restore an RDS database snapshot
+
+=end
+
+require 'methadone'
+require 'fog'
+
+module NewBackup
+
+  class MyRds
+
+    include Methadone::CLILogging
+
+    # Initialize the class with options
+    def initialize(options={})
+      debug "Options: #{options}"
+      @options = options
+    end
+
+
+    # Maximum number of tries to wait for database servers to be ready
+    MAX_TRIES = 3
+
+    # Restore a snapshot of the target database yielding the database to the block
+    def restore(&block)
+      connect do |connection|
+        get_rds(connection) do |rds_server|
+          retrieve_snapshot(rds_server) do |snapshot|
+            restore_db(connection,snapshot) do |db|
+              yield db
+            end
+          end
+        end
+      end
+    end
+
+    # Establishes a connection to AWS RDS and yeilds a block on the connection
+    def connect(&block)
+      raise "no block given in #{self.class}#connect" unless block_given?
+      aws = @options[:aws]
+      debug "AWS Options: #{aws}"
+      fog_options = {
+        :aws_access_key_id 	=> aws[:access_key],
+        :aws_secret_access_key 	=> aws[:secret_key],
+        :region 		=> aws[:rds_region]}
+
+      Fog.timeout = @options[:fog][:timeout]
+      yield Fog::AWS::RDS.new(fog_options)
+    end
+
+    # Get the RDS server
+    def get_rds(connection)
+      debug "rds instance_id: #{@options[:rds][:instance_id]}"
+      debug "Connection servers: #{connection.servers}"
+      rds_server = connection.servers.get(@options[:rds][:instance_id])
+      raise "No RDS server!" if rds_server.nil?
+      yield rds_server
+    end
+
+
+    # Retrieve a snapshot 
+    def retrieve_snapshot(rds_server, &block)
+      begin
+        rds_server.snapshots.new(:id => snap_name).save
+        new_snapshot = rds_server.snapshots.get(snap_name)
+        1.upto(MAX_TRIES) do |i|
+          debug "waiting for new snapshot, try ##{i}"
+          new_snapshot.wait_for { ready? }
+        end
+        
+        yield new_snapshot
+      ensure
+        unless @options[:debug]
+          new_snapshot.destroy unless new_snapshot.nil?
+        end
+      end
+      
+    end
+
+    # Restore the snapshot to a database
+    def restore_db(connection, snapshot, &block)
+      begin
+        connection.
+          restore_db_instance_from_db_snapshot(snapshot.id,
+                                               backup_server_id,
+                                               {"DBSubnetGroupName" => @options[:rds][:subnet_group],
+                                                 "DBInstanceClass" => @options[:rds][:instance_type]}
+                                               )
+        backup_server = connection.servers.get(backup_server_id)
+        1.upto(MAX_TRIES) do |i|
+          debug "waiting for backup server, try ##{i}"
+          backup_server.wait_for { ready? }
+        end
+
+        
+        yield MySqlCmds.new(backup_server.endpoint['Address'],
+                            @options[:mysql][:username],
+                            @options[:mysql][:password],
+                            @options[:mysql][:database],
+                            @options[:mysql][:obfuscate_script])
+                              
+
+      ensure
+        unless @options[:debug]
+          backup_server.destroy unless backup_server.nil?
+        end
+      end
+      
+    end
+
+
+
+    # Return the snapshot name
+    def snap_name
+      "s3-dump-snap-#{@options[:timestamp]}".tap{|t| debug "Snap Name: #{t}"}
+    end
+
+    # Return the backup server id
+    def backup_server_id
+      "#{@options[:rds][:instance_id]}-s3-dump-server-#{@options[:timestamp]}".tap{|t| debug "Backup Server ID: #{t}"}
+    end
+
+
+
+  end
+
+
+end

data/lib/new_backup/mys3.rb

@@ -0,0 +1,123 @@
+=begin
+
+= mys3.rb
+
+*Copyright*::    (C) 2013 by Novu, LLC
+*Author(s)*::    Tamara Temple <tamara.temple@novu.com>
+*Since*::        2013-05-01
+*License*::      MIT
+*Version*::      0.0.1
+
+== Description
+
+Upload the raw file and the clean file up to AWS S3
+
+=end
+
+require 'methadone'
+require 'debugger'
+
+module NewBackup
+
+  class MyS3
+
+    include Methadone::CLILogging
+
+    # Maximum number of attempts to try to upload file
+    MAX_TRIES = 3
+
+    # Initialize the MyS3 object
+    #
+    # options:: pass in the program options
+    def initialize(options={})
+      @options = options
+    end
+
+    # Connect to S3, cache the connection, and yield self
+    def connect(&block)
+
+      options = {
+        :aws_access_key_id => @options[:aws][:access_key],
+        :aws_secret_access_key => @options[:aws][:secret_key],
+        :region => @options[:s3][:region],
+        :provider => 'AWS',
+        :scheme => 'https'}
+      
+      connection = Fog::Storage.new(options)
+
+      yield connection
+      
+    end
+
+    # Connect to specific S3 bucket
+    #
+    # == inputs
+    # *connection*:: S3 connection to use
+    # *bucket_name*:: name of the bucket to use in S3
+    # *&block*:: block passed to evaluate in context of bucket
+    # (If no block given simply return the bucket pointer.)
+    def connect_bucket(connection, bucket_name, &block)
+      debug "#{self.class}##{__method__}:#{__LINE__}: connection: #{connection.inspect}"
+      buckets = connection.directories
+      raise "No buckets!" if buckets.nil? || buckets.empty?
+      bucket = connection.directories.get(bucket_name)
+      raise "No bucket #{bucket_name}" if bucket.nil?
+
+      if block_given?
+        yield bucket
+      else
+        bucket
+      end
+      
+    end
+
+
+
+
+
+    # Do the heavy lifing to put the file in the appropriate bucket
+    #
+    # bucket:: directory where to put the file
+    # fn:: name of file to upload (gzipped)
+    def put_file(bucket,fn)
+      s3_fn = File.join(@options[:s3][:prefix], File.basename(fn))
+      options = {
+        :key => s3_fn,
+        :body => File.open(fn,'r'),
+        :acl => 'authenticated-read',
+        :encryption => 'AES256',
+        :content_type => 'application/x-gzip'}
+      
+      bucket.files.create(options)
+
+      # Verify that file was uploaded
+      files = bucket.files.all.select {|f| f.key == s3_fn }
+      raise "#{fn} was not uploaded to #{s3_fn}!" unless files.count > 0
+      "s3://#{bucket.key}/#{s3_fn}"
+
+    end
+
+    # Prune extra files
+    def prune(bucket, keep=0)
+
+      if keep > 0
+        files = bucket.files.all('prefix' => @options[:s3][:prefix])
+
+        return if files.nil? || files.empty?
+
+        if files.count > keep
+          files.sort {|x,y| x.last_modified <=> y.last_modified}.
+            take(files.count - keep).
+            map(&:destroy)
+        end
+
+      end
+      
+      files = bucket.files.all('prefix' => @options[:s3][:prefix])
+      files.count
+
+    end
+
+  end
+
+end