ey_cloud_server 1.3.1 → 1.4.5.pre

data/lib/ey-flex/snapshot_minder.rb

@@ -13,20 +13,20 @@ module EY
         opts.banner = "Usage: ey-snapshots [-flag] [argument]"
         opts.define_head "ey-snapshots: managing your snapshots..."
         opts.separator '*'*80
-        
-        opts.on("-l", "--list-snapshots", "list snapshots") do 
+
+        opts.on("-l", "--list-snapshots", "list snapshots") do
           options[:command] = :list_snapshots
         end
 
         opts.on("-c", "--config CONFIG", "Use config file.") do |config|
           options[:config] = config
         end
-        
+
         opts.on("-i", "--instance-id ID", "specify the instance id to work with(only needed if you are running this from ourside of ec2)") do |iid|
           options[:instance_id] = iid
         end
-        
-        
+
+
         opts.on("--snapshot", "take snapshots of both of your volumes(only runs on your ec2 instance)") do
           options[:command] = :snapshot_volumes
         end
@@ -34,7 +34,7 @@ module EY
         opts.on("-q", "--quiet", "Supress output to STDOUT") do
           options[:quiet] = true
         end
-        
+
       end
 
       opts.parse!(args)
@@ -58,7 +58,7 @@ module EY
       get_instance_id
       silence_stream($stderr) { find_volume_ids }
     end
-    
+
     def find_volume_ids
       @volume_ids = {}
       @ec2.describe_volumes.each do |volume|
@@ -67,26 +67,26 @@ module EY
             @volume_ids[:data] = volume[:aws_id]
           elsif volume[:aws_device] == "/dev/sdz2"
             @volume_ids[:db] = volume[:aws_id]
-          end    
+          end
         end
       end
       say "Volume IDs are #{@volume_ids.inspect}"
       @volume_ids
     end
-    
+
     def list_snapshots
       @snapshot_ids = {}
       @ec2.describe_snapshots.sort { |a,b| b[:aws_started_at] <=> a[:aws_started_at] }.each do |snapshot|
         @volume_ids.each do |mnt, vol|
           if snapshot[:aws_volume_id] == vol
             (@snapshot_ids[mnt] ||= []) << snapshot[:aws_id]
-          end  
+          end
         end
       end
       say "Snapshots #{@snapshot_ids.inspect}"
       @snapshot_ids
     end
-    
+
     def clean_snapshots(keep=5)
       list_snapshots
       @snapshot_ids.each do |mnt, ids|
@@ -101,7 +101,7 @@ module EY
       end
       list_snapshots
     end
-    
+
     def snapshot_volumes
       snaps = []
       @volume_ids.each do |vol, vid|
@@ -120,10 +120,10 @@ module EY
       end
       snaps
     end
-    
+
     def get_instance_id
       return @instance_id if @instance_id
-      
+
       open('http://169.254.169.254/latest/meta-data/instance-id') do |f|
         @instance_id = f.gets
       end
@@ -131,14 +131,22 @@ module EY
       say "Instance ID is #{@instance_id}"
       @instance_id
     end
-    
+
     def sync_filesystem_buffers
       sync_cmd = "sync && sync && sync"
       system(sync_cmd)
     end
-    
+
     def create_snapshot(volume_id)
-      snap = @ec2.create_snapshot(volume_id)
+      retries = 0
+      begin
+        snap = @ec2.create_snapshot(volume_id)
+      rescue RightAws::AwsError
+        retries += 1
+        raise if retries > 10
+        sleep retries * retries
+        retry
+      end
       say "Created snapshot of #{volume_id} as #{snap[:aws_id]}"
       snap
     end
@@ -158,11 +166,11 @@ module EY
         stream.reopen(old_stream)
       end
   end
-  
+
   class Mysql
-    
+
     attr_accessor :dbh
-    
+
     def initialize(username, password, lock_wait_timeout)
       @username = username
       @password = password
@@ -170,36 +178,36 @@ module EY
       @lock_wait_timeout = lock_wait_timeout.nil? ? 5 : lock_wait_timeout
       @master_status_file = "/db/mysql/.snapshot_backup_master_status.txt"
     end
-    
+
     def waiting_read_lock_thread
-      thread_cmd = "mysql -p#{@password} -u #{@username} -N -e 'show full processlist;' | grep 'flush tables with read lock' | awk '{print $1}'" 
+      thread_cmd = "mysql -p#{@password} -u #{@username} -N -e 'show full processlist;' | grep 'flush tables with read lock' | awk '{print $1}'"
       %x{#{thread_cmd}}
     end
-    
+
     def write_master_status
       master_status_cmd = "mysql -p#{@password} -u #{@username} -e'SHOW MASTER STATUS\\G' > #{@master_status_file}"
       system(master_status_cmd)
     end
-    
+
     def flush_tables_with_read_lock
       pipe = IO.popen("mysql -u #{@username} -p#{@password}", 'w')
       @read_lock_pid = pipe.pid
-      
+
       pipe.puts('flush tables with read lock;')
       sleep(@lock_wait_timeout)
-      
+
       if (thread_id = waiting_read_lock_thread) != ''
         Process.kill('TERM', @read_lock_pid)
-        
+
         # after killing the process the mysql thread is still hanging out, need to kill it directly
-        kill_thread_cmd = "mysql -u #{@username} -p#{@password} -e'kill #{thread_id};'" 
+        kill_thread_cmd = "mysql -u #{@username} -p#{@password} -e'kill #{thread_id};'"
         system(kill_thread_cmd)
         abort "Read lock not acquired after #{@lock_wait_timeout} second timeout.  Killed request and aborting backup."
       end
-      
+
       true
     end
-    
+
     def unlock_tables
       # technically we don't actually have to do anything here since the spawned
       # process that has the read lock will die with this one but it doesn't hurt
@@ -207,7 +215,7 @@ module EY
       Process.kill('TERM', @read_lock_pid)
       true
     end
-    
+
     def disconnect
       @dbh.disconnect
     end

data/lib/ey-flex/version.rb

@@ -1,3 +1,3 @@
 module EY::CloudServer
-  VERSION = '1.3.1'
+  VERSION = '1.4.5.pre'
 end

data/lib/ey_backup.rb

@@ -0,0 +1,96 @@
+require File.expand_path(__FILE__ + '/../ey-flex')
+require 'logger'
+require 'forwardable'
+require 'stringio'
+require 'open4'
+require 'zlib'
+require 'ostruct'
+require 'fileutils'
+
+module EY
+  module Backup
+    extend self
+    attr_accessor :logger, :bucket_minder, :engine, :config
+
+    def run(argv = ARGV)
+      options = CLI.run(argv)
+      setup(options)
+      dispatch(options)
+    end
+
+    def new_backup(options = {})
+      Dumper.run(options)
+    end
+
+    def list(options = {})
+      if options[:db].nil? || options[:db].empty?
+        BackupSet.list_all
+      else
+        #list a single database's backups
+        BackupSet.list(options[:db])
+      end
+    end
+
+    def restore(options = {})
+      Loader.run(options)
+    end
+
+    def download(options = {})
+      Loader.download(options)
+    end
+
+    def setup(options)
+      setup_config(options)
+      setup_logger(options)
+      setup_minder(options)
+      setup_engine(options)
+    end
+
+    def setup_config(options)
+      @config = OpenStruct.new(options)
+    end
+
+    def setup_logger(options)
+      if options[:quiet]
+        @logger ||= Logger.quiet
+      else
+        @logger ||= Logger.new
+      end
+    end
+
+    def setup_minder(options)
+      @bucket        = "ey-backup-#{Digest::SHA1.hexdigest(options[:aws_secret_id])[0..11]}"
+      @bucket_minder = EY::BucketMinder.new(options.merge(:bucket => @bucket))
+    end
+
+    def setup_engine(options)
+      @engine = Engine.lookup(options[:engine]).new(options[:dbuser], options[:dbpass])
+    end
+
+    def dispatch(options)
+      send(options[:command], options)
+    end
+
+    def base_path
+      "/mnt/tmp"
+    end
+  end
+end
+
+
+lib_dir = File.expand_path(__FILE__ + '/../ey_backup')
+require lib_dir + '/spawner'
+require lib_dir + '/logger'
+require lib_dir + '/base'
+require lib_dir + '/backup_set'
+require lib_dir + '/cli'
+require lib_dir + '/dumper'
+require lib_dir + '/engine'
+require lib_dir + '/loader'
+
+require lib_dir + '/engines/mysql_engine'
+require lib_dir + '/engines/postgresql_engine'
+
+require lib_dir + '/processors/gzipper'
+require lib_dir + '/processors/splitter'
+require lib_dir + '/processors/gpg_encryptor'

data/lib/ey_backup/backup_set.rb

@@ -0,0 +1,128 @@
+module EY
+  module Backup
+    class BackupSet < Struct.new(:name, :keys)
+      include EY::Backup::Logging
+      extend  EY::Backup::Logging
+
+      attr_accessor :files
+
+      MULTIPART_EXTENSION = /\.part\d+/
+
+      def self.parse(objects)
+        objects.map {|o| new(o[:name], o[:keys]) }
+      end
+
+      def self.from(database, file)
+        backup = new(File.basename(file), [])
+        backup.files = [file]
+        backup
+      end
+
+      def self.cleanup(database, keep)
+        say "Cleanup for #{database}"
+        backups_for(database)[0...(-keep)].each do |backup|
+          say "deleting: #{backup.name}"
+          backup.delete!
+        end
+      end
+
+      def self.list_all
+        list('all', backups)
+      end
+
+      def self.backups
+        EY::Backup.config.databases.map {|db| backups_for(db) }.flatten
+      end
+
+      def self.list(database, backups = backups_for(database))
+        info "Listing database backups for #{database}"
+
+        info "#{backups.size} backup(s) found"
+
+        backups.each_with_index do |db, i|
+          info "#{i}:#{database} #{db.normalized_name}"
+        end
+
+        backups
+      end
+
+      def self.backups_for(database_name)
+        s3_objects = EY::Backup.bucket_minder.list(:prefix => "#{EY::Backup.config.environment}.#{database_name}", :print => false)
+        parse(s3_objects).sort
+      end
+
+      def self.upload(database, file)
+        EY::Backup.bucket_minder.name = "#{EY::Backup.config.environment}.#{database}/#{File.basename(file)}"
+        EY::Backup.bucket_minder.upload_object(file, false)
+      end
+
+      def self.download(database, index)
+        fatal "You didn't specify a database name: e.g. 1:rails_production" if database.empty? || index.empty?
+
+        backup = backups_for(database)[index.to_i]
+
+        fatal "No backup found for database #{database}: requested index: #{index}" unless backup
+
+        backup.download!
+
+        backup
+      end
+
+      def initialize(*)
+        super
+
+        @files = []
+      end
+
+      def parts
+        [keys.size, files.size].max
+      end
+
+      def normalized_name
+        normalize(name)
+      end
+
+      def upload!(database)
+        files.each do |file|
+          EY::Backup.bucket_minder.name = "#{EY::Backup.config.environment}.#{database}/#{File.basename(file)}"
+          EY::Backup.bucket_minder.upload_object(file, false)
+          info "successful upload: #{file}"
+        end
+      end
+
+      def download!
+        keys.each do |key|
+          info "Downloading #{key}"
+          filename = File.join(EY::Backup.base_path, normalize(key))
+          File.open(filename, 'wb') do |f|
+            EY::Backup.bucket_minder.stream(key) do |chunk|
+              f << chunk
+            end
+          end
+
+          files << filename
+        end
+      end
+
+      def delete!
+        keys.each do |key|
+          EY::Backup.bucket_minder.remove_object(key)
+        end
+      end
+
+      def rm!
+        files.each do |file|
+          FileUtils.rm(file)
+        end
+      end
+
+      def normalize(o)
+        o.gsub(/^.*?\//, '')
+      end
+
+      def <=>(o)
+        self.name <=> o.name
+      end
+    end
+  end
+end

data/lib/ey_backup/base.rb

@@ -0,0 +1,18 @@
+module EY
+  module Backup
+    class Base
+      extend Forwardable
+
+      def_delegators :logger, :fatal, :error, :warn, :info, :debug, :say
+
+      def initialize(options = {})
+        @environment = options[:env]
+        @engine_name = options[:engine]
+      end
+
+      def logger
+        EY::Backup.logger
+      end
+    end
+  end
+end

data/lib/ey_backup/cli.rb

@@ -0,0 +1,89 @@
+module EY
+  module Backup
+    module CLI
+      extend self
+
+      def run(argv)
+        options = default_options.merge(opt_parse(argv))
+
+        config_path = options[:config] || "/etc/.#{options[:engine]}.backups.yml"
+
+        config_for(config_path).merge(options)
+      end
+
+      def default_options
+        {
+          :command => :new_backup,
+          :format => "gzip",
+          :engine => 'mysql',
+        }
+      end
+
+      def opt_parse(argv)
+        # Build a parser for the command line arguments
+        options = {}
+
+        opts = OptionParser.new do |opts|
+          opts.version = EY::CloudServer::VERSION
+
+          opts.banner = "Usage: eybackup [-flag] [argument]"
+          opts.define_head "eybackup: backing up your shit since way back when..."
+          opts.separator '*'*80
+
+          opts.on("-l", "--list-backup DATABASE", "List mysql backups for DATABASE") do |db|
+            options[:db] = (db || 'all')
+            options[:command] = :list
+          end
+
+          opts.on("-n", "--new-backup", "Create new mysql backup") do
+            options[:command] = :new_backup
+          end
+
+          opts.on("-c", "--config CONFIG", "Use config file.") do |config|
+            options[:config] = config
+          end
+
+          opts.on("-d", "--download BACKUP_INDEX", "download the backup specified by index. Run eybackup -l to get the index.") do |index|
+            options[:command] = :download
+            options[:index] = index
+          end
+
+          opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres.") do |engine|
+            options[:engine] = engine
+          end
+
+          opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index WARNING! will overwrite the current db with the backup. Run eybackup -l to get the index.") do |index|
+            options[:command] = :restore
+            options[:index] = index
+          end
+
+          opts.on("-k", "--key KEYID", "Public key ID to use for the backup operation") do |key_id|
+            options[:key_id] = key_id
+          end
+
+          opts.on("-q", "--quiet", "Supress output to STDOUT") do
+            options[:quiet] = true
+          end
+
+          opts.on("-s", "--split_size INTEGER", "Maximum size of a single backup file before splitting.") do |split_size|
+            options[:split_size] = split_size.to_i
+          end
+        end
+
+        opts.parse!(argv)
+
+        options
+      end
+
+      def config_for(filename)
+        if File.exist?(filename)
+          config = YAML::load(File.read(filename))
+          config[:environment] ||= config[:env]
+          config
+        else
+          abort "You need to have a backup file at #{filename}"
+        end
+      end
+    end
+  end
+end