ey_cloud_server 1.4.47 → 1.4.49

checksums.yaml

@@ -1,7 +1,7 @@
---- 
-SHA512: 
-  metadata.gz: 8ea9f9720c697f5544e918ffc6efaa33294ee3b1ba9c5bf3b79e7dabb8fb7215aae5e9daea553db1d570785e0f86dc8522f81d2ad7ff8b720f3a7d53c523583d
-  data.tar.gz: a2e1b80bc70a1a6c01ec14de5691f467a8cd1fe443155220fbfed4e58ddbed7d87543e2a84978c781b06991ecd482fd7759aa5b4c912dd7a2122f4f352c71cf9
-SHA1: 
-  metadata.gz: eae0875e369ab43aba8d8f8f8df4f73c718afcfb
-  data.tar.gz: 959fdaab084788b57dc364842f57a8a90b3e9e44
+---
+SHA1:
+  metadata.gz: b322ebd431ff9f4bd27a8a908d145ad4e70f01e9
+  data.tar.gz: 4a67d4081f42712cd36e8d683f1b00f767196d7a
+SHA512:
+  metadata.gz: 25bf1195e76d0042b379f7bc3aa9fb21e7ff30b5ab8b0b93cabc698c54032ec8dc15a20cd3b163e3d314fda03404310ca4e437fd2612d4d87e0714469b791499
+  data.tar.gz: 1211cea6558f7b4d1639bc686d24c015522fd35a69f55733b6cc7091a2a7e66ae2b920e597e48b009d99734701c19301d5188a306989f15074fe0b02744bb0d7

data/features/support/env.rb

@@ -3,15 +3,15 @@ require File.dirname(__FILE__) + "/../../lib/ey_backup"
 require 'fakeweb'
 require 'fakefs/safe'
 require 'randexp'
+require 'fog/aws'
+
+Fog.mock!
 
 require File.dirname(__FILE__) + '/../../spec/fakefs_hax'
 require File.dirname(__FILE__) + '/../../spec/helpers'
 
 FakeWeb.allow_net_connect = false # if it's not here, it doesn't exist
 Randexp::Dictionary.words
-Fog.mock!
-
-Fog.mock!
 
 World(Helpers)
 

data/lib/ey-flex.rb

@@ -6,7 +6,6 @@ require 'fileutils'
 require 'json/ext'
 require 'open-uri'
 require 'rest_client'
-require 'dbi'
 require 'zlib'
 require 'stringio'
 require 'yaml'
@@ -29,7 +28,8 @@ module EY
     @enzyme_api ||= EY::Enzyme::API.new(
       enzyme_config[:api],
       enzyme_config[:instance_id],
-      enzyme_config[:token]
+      enzyme_config[:token],
+      EY::Backup.logger
     )
   end
 
@@ -43,7 +43,6 @@ module EY
   module CloudServer; end
 end
 
-require lib_dir + '/big-brother'
 require lib_dir + '/bucket_minder'
 require lib_dir + '/ey-api'
 require lib_dir + '/snapshot_minder'

data/lib/ey-flex/snapshot_minder.rb

@@ -29,7 +29,7 @@ module EY
         def list
           @commands_called << "list_snapshots"
           [EY::InstanceAPIClient::Snapshot.new({
-            'state' => 'in progres',
+            'state' => 'in progress',
             'progress' => '0%',
             'volume_type' => 'db',
             'snapshot_id' => 12,

data/lib/ey_backup.rb

@@ -6,7 +6,8 @@ require 'open4'
 require 'zlib'
 require 'ostruct'
 require 'fileutils'
-require 'fog'
+require 'fog/aws'
+require 'timeout'
 
 module Fog
   module AWS
@@ -72,6 +73,7 @@ module EY
 
       def restore
         databases.each do |database|
+          @engine.check_if_replica
           Loader.run(database, @options[:index])
         end
       end
@@ -110,6 +112,10 @@ module EY
           EY::Backup.logger ||= Logger.quiet
         else
           EY::Backup.logger ||= Logger.new
+
+          if @options[:verbose]
+            EY::Backup.logger.set_verbose
+          end
         end
       end
 
@@ -130,7 +136,7 @@ module EY
         if ! @options.key?(:dbhost) or @options[:dbhost] == nil or @options[:dbhost] == ""
           @options[:dbhost] = 'localhost'
         end
-        @engine = engine_class.new(@options[:dbuser], @options[:dbpass], @options[:dbhost], @options[:key_id])
+        @engine = engine_class.new(@options[:dbuser], @options[:dbpass], @options[:dbhost], @options[:key_id], @options[:force])
       end
 
       def dispatch

data/lib/ey_backup/backend.rb

@@ -1,4 +1,4 @@
-require 'fog'
+require 'fog/aws'
 
 module EY
   module Backup

data/lib/ey_backup/base.rb

@@ -3,7 +3,7 @@ module EY
     class Base
       extend Forwardable
 
-      def_delegators :logger, :fatal, :error, :warn, :info, :debug, :say
+      def_delegators :logger, :fatal, :error, :warn, :info, :verbose, :debug, :say
 
       def logger
         EY::Backup.logger

data/lib/ey_backup/cli.rb

@@ -5,6 +5,7 @@ module EY
 
       def run(argv)
         options = default_options.merge(opt_parse(argv))
+        verify_restore if options[:command] == :restore and !options[:force]
 
         config_path = options[:config] || "/etc/.#{options[:engine]}.backups.yml"
 
@@ -16,8 +17,18 @@ module EY
           :command => :new_backup,
           :format => "gzip",
           :engine => 'mysql',
+          :verbose => false,
         }
       end
+      
+      def verify_restore
+        res = ''
+        puts "This will overwrite your database, are you sure you would like to proceed (Y/n)?"
+        Timeout::timeout(30){
+          res = gets.strip
+        }
+        abort("You indicated '#{res}', exiting!") unless res.upcase == 'Y'
+      end
 
       def opt_parse(argv)
         # Build a parser for the command line arguments
@@ -25,12 +36,13 @@ module EY
 
         opts = OptionParser.new do |opts|
           opts.version = EY::CloudServer::VERSION
+          puts '' # add a blank line
 
           opts.banner = "Usage: eybackup [-flag] [argument]"
-          opts.define_head "eybackup: manage dump (mysqldump/pg_dump) style backups of your database."
+          opts.define_head "  eybackup: manage dump (mysqldump/pg_dump) style backups of your database."
           opts.separator '*'*80
 
-          opts.on("-l", "--list-backup DATABASE", "List mysql backups for DATABASE") do |db|
+          opts.on("-l", "--list-backup DATABASE", "List backups for DATABASE") do |db|
             if db == "all"
               db = nil
             end
@@ -38,7 +50,10 @@ module EY
             options[:command] = :list
           end
 
-          opts.on("-n", "--new-backup", "Create new mysql backup") do
+          opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgresql.") do |engine|
+            options[:engine] = engine
+          end
+          opts.on("-n", "--new-backup", "Create a new backup (default)") do
             options[:command] = :new_backup
           end
 
@@ -53,36 +68,49 @@ module EY
           opts.on("-t", "--tmp_dir TMPDIR", "Use the given directory for temporary storage.") do |tmp_dir|
             options[:tmp_dir] = tmp_dir
           end
+          
+          opts.on("-k", "--key KEYID", "Public key ID to use for the backup operation") do |key_id|
+            options[:key_id] = key_id
+          end
+
+          opts.on("-q", "--quiet", "Suppress output to STDOUT") do
+            options[:quiet] = true
+          end
+
+          opts.on("-s", "--split_size INTEGER", "Maximum size of a single backup file before splitting.") do |split_size|
+            options[:split_size] = split_size.to_i
+          end
 
-          opts.on("-d", "--download BACKUP_INDEX", "download the backup specified by index. Run eybackup -l to get the index.") do |index_and_db|
+  	      opts.on("-v", "--verbose", "Show verbose output") do
+  	        options[:verbose] = true
+  	      end
+
+          opts.on("-d", "--download BACKUP_INDEX", 
+                  "Download the backup specified by index.",
+                  '  Run `eybackup -l #{db_name}` to get the index.',
+                  '  BACKUP_INDEX uses the format #{index_number}:#{db_name}') do |index_and_db|
             options[:command] = :download
             db, index = split_index(index_and_db)
             options[:index] = index
             options[:db] = db
           end
 
-          opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres.") do |engine|
-            options[:engine] = engine
-          end
-
-          opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index WARNING! will overwrite the current db with the backup. Run eybackup -l to get the index.") do |index_and_db|
+          opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index.",
+                  "  **WARNING!** will overwrite the current db with the backup.",
+                  '  Run `eybackup -l #{db_name}` to get the index.',
+                  '  BACKUP_INDEX uses the format #{index_number}:#{db_name}') do |index_and_db|
             options[:command] = :restore
             db, index = split_index(index_and_db)
             options[:index] = index
             options[:db] = db
           end
-
-          opts.on("-k", "--key KEYID", "Public key ID to use for the backup operation") do |key_id|
-            options[:key_id] = key_id
+          
+          options[:force] = false
+          opts.on("-f", "--force", "Force backup restore, bypass confirmation prompts.",
+                  "  For use with automated restore operations (e.g. Staging).") do
+            options[:force] = true
           end
 
-          opts.on("-q", "--quiet", "Supress output to STDOUT") do
-            options[:quiet] = true
-          end
-
-          opts.on("-s", "--split_size INTEGER", "Maximum size of a single backup file before splitting.") do |split_size|
-            options[:split_size] = split_size.to_i
-          end
         end
 
         opts.parse!(argv)

data/lib/ey_backup/database.rb

@@ -11,8 +11,9 @@ module EY
         @backend = backend
         @environment = environment
         @name = name
+        @backup_size = nil
       end
-      attr_reader :name, :engine, :environment, :keep, :base_path
+      attr_reader :name, :engine, :environment, :keep, :base_path, :backup_size
 
       def bucket_minder
         @backend.bucket_minder
@@ -28,6 +29,7 @@ module EY
 
         basename = generate_basename
         file = @engine.dump(@name, basename)
+        @backup_size = number_to_human_size(File.size(file))
 
         BackupSet.from(self, basename, file)
       end
@@ -43,6 +45,18 @@ module EY
       def timestamp
         Time.now.strftime("%Y-%m-%dT%H-%M-%S")
       end
+      
+      def number_to_human_size(size)
+        if size < 1024
+          "#{size} bytes"
+        elsif size < 1024.0 * 1024.0
+          "%.01f KB" % (size / 1024.0)
+        elsif size < 1024.0 * 1024.0 * 1024.0
+          "%.01f MB" % (size / 1024.0 / 1024.0)
+        else
+          "%.01f GB" % (size / 1024.0 / 1024.0 / 1024.0)
+        end
+      end
     end
   end
 end

data/lib/ey_backup/dumper.rb

@@ -1,14 +1,40 @@
 module EY
   module Backup
     class Dumper < Base
+      require 'shellwords'
       include Logging
 
       attr_reader :database
 
       def self.run(databases, split_size)
+        exceptions = []
+        completed = []
         databases.each do |database|
-          new(database).run(split_size)
+          begin
+            backup_size = new(database).run(split_size)
+            completed << "#{database.name} (#{backup_size})"
+          rescue => e
+            puts e
+            exceptions << database.name
+            next
+          end
         end
+        message = ''
+        message = message + "Failures: #{exceptions}" if ! exceptions.empty?
+        message = message + ", " if ! exceptions.empty? and ! completed.empty?
+        message = message + "Completed successfully: #{completed}" if ! completed.empty?
+        
+        alert_level=exceptions.empty? ? 'OKAY' : 'FAILURE'
+        
+        message.gsub!("\n", '\n')
+        full_txt= "Severity: #{alert_level}\n" \
+                + "Time: #{Time.now.to_i}\n" \
+                + "Type: process-dbbackup summary\n" \
+                + "Plugin: exec\n" \
+                + "raw_message: '#{message}'"
+        full_txt = Shellwords.escape(full_txt)
+        alert_command = %Q(echo #{full_txt} | /engineyard/bin/ey-alert.rb 2>&1)
+        system(alert_command)
       end
 
       def initialize(database)
@@ -16,16 +42,18 @@ module EY
       end
 
       def run(split_size)
-        info "Doing database: #{@database.name}"
+        info("Doing database: #{@database.name}")
 
         backup_set = @database.dump
         backup_set.split!(split_size)
         backup_set.upload!
 
-        info "Successful backup: #{@database.name}.#{backup_set.normalized_name}"
+        okay("Successful backup: #{@database.name} (#{@database.backup_size})", @database.name)
         backup_set.cleanup
         backup_set.rm!
+        @database.backup_size
       end
+      
     end
   end
 end

data/lib/ey_backup/engine.rb

@@ -3,7 +3,7 @@ module EY
     class Engine < Base
       include Spawner
 
-      attr_reader :username, :password, :host, :key_id
+      attr_reader :username, :password, :host, :key_id, :force
 
       def self.label
         @label
@@ -26,8 +26,8 @@ module EY
           EY::Backup.logger.fatal("Unknown database engine: #{label}")
       end
 
-      def initialize(username, password, host, key_id)
-        @username, @password, @host, @key_id  = username, password, host, key_id
+      def initialize(username, password, host, key_id, force)
+        @username, @password, @host, @key_id, @force  = username, password, host, key_id, force
       end
 
       def gpg?

data/lib/ey_backup/engines/mysql_engine.rb

@@ -6,7 +6,7 @@ module EY
       def dump(database_name, basename)
         file = basename + '.sql'
 
-        command = "mysqldump #{username_option} #{host_option} #{password_option} #{single_transaction_option(database_name)} #{routines_option} #{database_name}"
+        command = "mysqldump #{username_option} #{host_option} #{single_transaction_option(database_name)} #{routines_option} #{master_data_option} #{databases_option(database_name)} 2> /tmp/eybackup.$$.dumperr"
 
         if gpg?
           command << " | " << GPGEncryptor.command_for(key_id)
@@ -18,12 +18,13 @@ module EY
 
         command << " > #{file}"
 
-        run(command)
+        run(command, database_name)
 
         file
       end
 
       def load(database_name, file)
+        cycle_database(database_name)
         command = "cat #{file}"
 
         if gpg?
@@ -32,17 +33,30 @@ module EY
           command << " | " << GZipper.gunzip
         end
 
-        command << " | mysql #{username_option} #{host_option} #{password_option} #{database_name}"
+        command << " | mysql #{username_option} #{host_option} #{database_name}"
 
         run(command)
       end
+      
+      def check_if_replica
+        err_msg="ERROR: Target host: '#{host}' is currently: "
+        err_msg=err_msg + "a replica based on 'show slave status', " if db_replicating?
+        err_msg=err_msg + "in read_only mode, " if read_only_on?
+        err_msg=err_msg + "restores should be processed against the master."
+        
+        EY::Backup.logger.fatal(%Q{#{err_msg}}) if db_replicating? or read_only_on?
+      end
 
       def routines_option
         "--routines"
       end
-
-      def password_option
-        "-p'#{password}'" unless password.nil? || password.empty?
+      
+      def master_data_option
+        "--master-data=2" if log_bin_on?
+      end
+      
+      def databases_option(db)
+        "--add-drop-database --databases #{db}"
       end
 
       def host_option
@@ -59,9 +73,37 @@ module EY
 
       def db_has_myisam?(database_name)
         query, stdout = "SELECT 1 FROM information_schema.tables WHERE table_schema='#{database_name}' AND engine='MyISAM' LIMIT 1;", StringIO.new
-        spawn(%Q{mysql #{username_option} #{password_option} #{host_option} -N -e"#{query}"}, stdout)
+        spawn(%Q{mysql #{username_option} #{host_option} -N -e"#{query}"}, stdout)
         stdout.string.strip == '1'
       end
+      
+      def db_replicating?
+        stdout = StringIO.new
+        spawn(%Q{mysql #{username_option} #{host_option} -BN -e"show slave status"|wc -l}, stdout)
+        stdout.string.to_i > 0
+      end
+      
+      def read_only_on?
+        stdout = StringIO.new
+        spawn(%Q{mysql #{username_option} #{host_option} -BN -e"select @@global.read_only"}, stdout)
+        stdout.string.to_i == 1
+      end
+      
+      def log_bin_on?
+        stdout = StringIO.new
+        spawn(%Q{mysql #{username_option} #{host_option} -BN -e"select @@global.log_bin"}, stdout)
+        stdout.string.to_i == 1
+      end
+      
+      def cycle_database(database_name)
+        query = "show create database #{database_name} \\G"
+        create_cmd = %x(mysql #{username_option} #{host_option} -NB -e "#{query}"|tail -n 1)
+        create_cmd="Create database #{database_name}" if create_cmd==""
+        
+        %x(mysql #{username_option} #{host_option} -e 'DROP DATABASE IF EXISTS #{database_name}')
+        %x(mysql #{username_option} #{host_option} -e '#{create_cmd}')
+        
+      end
 
       def suffix
         /sql\.(gz|gpz)$/