ey_cloud_server 1.4.5 → 1.4.26

data/lib/ey_backup/database.rb

@@ -0,0 +1,48 @@
+module EY
+  module Backup
+    class Database
+      def initialize(engine, base_path, keep, backend, environment, name)
+        if name.nil? || name.empty?
+          raise ArgumentError, "database name is blank"
+        end
+        @engine    = engine
+        @base_path = base_path
+        @keep      = keep
+        @backend = backend
+        @environment = environment
+        @name = name
+      end
+      attr_reader :name, :engine, :environment, :keep, :base_path
+
+      def bucket_minder
+        @backend.bucket_minder
+      end
+
+      def backups
+        s3_objects = bucket_minder.list("#{@environment}.#{@name}").select { |o| o[:name] =~ @engine.suffix }
+        s3_objects.map {|o| BackupSet.new(self, o[:name], nil, o[:keys]) }.sort
+      end
+
+      def dump
+        FileUtils.mkdir_p(@base_path)
+
+        basename = generate_basename
+        file = @engine.dump(@name, basename)
+
+        BackupSet.from(self, basename, file)
+      end
+
+      def start_upload(filenames)
+        @backend.start_upload(filenames, @environment, @name)
+      end
+
+      def generate_basename
+        "#{@base_path}/#{@name}.#{timestamp}"
+      end
+
+      def timestamp
+        Time.now.strftime("%Y-%m-%dT%H-%M-%S")
+      end
+    end
+  end
+end

data/lib/ey_backup/dumper.rb

@@ -1,47 +1,31 @@
 module EY
   module Backup
     class Dumper < Base
-      attr_accessor :database
+      include Logging
 
-      def_delegators EY::Backup, :engine, :minder, :base_path
+      attr_reader :database
 
-      def self.run(options = {})
-        if database = options[:db]
-          new(database, options).run
-        else
-          options[:databases].each do |database|
-            new(database, options).run
-          end
+      def self.run(databases, split_size)
+        databases.each do |database|
+          new(database).run(split_size)
         end
       end
 
-      def initialize(database, options = {})
-        super(options)
-        @database = database
+      def initialize(database)
+        @database  = database
       end
 
-      def run
-        info "doing database: #{database}"
+      def run(split_size)
+        info "Doing database: #{@database.name}"
 
-        FileUtils.mkdir_p('/mnt/tmp')
+        backup_set = @database.dump
+        backup_set.split!(split_size)
+        backup_set.upload!
 
-        backup = engine.dump(database, base_name)
-
-        backup.upload!(database)
-
-        info "Successful backup: #{database}.#{backup.normalized_name}"
-        BackupSet.cleanup(database, EY::Backup.config.keep)
-      end
-
-
-      def base_name
-        "#{base_path}/#{database}.#{timestamp}"
-      end
-
-      def timestamp
-        @timestamp ||= Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')
+        info "Successful backup: #{@database.name}.#{backup_set.normalized_name}"
+        backup_set.cleanup
+        backup_set.rm!
       end
-
     end
   end
 end

data/lib/ey_backup/engine.rb

@@ -3,7 +3,7 @@ module EY
     class Engine < Base
       include Spawner
 
-      attr_accessor :username, :password
+      attr_reader :username, :password, :host
 
       def self.label
         @label
@@ -26,25 +26,15 @@ module EY
           EY::Backup.logger.fatal("Unknown database engine: #{label}")
       end
 
-      def initialize(username, password)
-        @username, @password = username, password
-      end
-
-      def post_dump_process(backup)
-        processors.inject(backup) do |backup, processor|
-          processor.dump(backup)
-        end
-      end
-
-      def pre_load_process(backup)
-        processors.reverse.inject(backup) do |backup, processor|
-          processor.load(backup)
-        end
+      def initialize(username, password, host, key_id)
+        @username, @password, @host = username, password, host
+        @key_id = key_id
       end
+      attr_reader :key_id
 
       def gpg?
-        not EY::Backup.config.key_id.nil? &&
-          EY::Backup.config.key_id.blank?
+        not @key_id.nil? &&
+          @key_id.blank?
       end
     end
   end

data/lib/ey_backup/engines/mysql_engine.rb

@@ -3,13 +3,13 @@ module EY
     class MysqlEngine < Engine
       register 'mysql'
 
-      def dump(database, basename)
+      def dump(database_name, basename)
         file = basename + '.sql'
 
-        command = "mysqldump #{username_option} #{password_option} #{single_transaction_option(database)} #{database}"
+        command = "mysqldump #{username_option} #{host_option} #{password_option} #{single_transaction_option(database_name)} #{routines_option} #{database_name}"
 
         if gpg?
-          command << " | " << GPGEncryptor.command
+          command << " | " << GPGEncryptor.command_for(key_id)
           file << GPGEncryptor.extension
         else
           command << " | " << GZipper.gzip
@@ -20,44 +20,52 @@ module EY
 
         run(command)
 
-        Splitter.dump(BackupSet.from(database, file))
+        file
       end
 
-      def load(database, backup)
-        backup = Splitter.load(backup)
-
-        command = "cat #{backup.files.first}"
+      def load(database_name, file)
+        command = "cat #{file}"
 
         if gpg?
-          command << " | " << GPGEncryptor.load_command
+          raise "Cannot load a GPG backup"
         else
           command << " | " << GZipper.gunzip
         end
 
-        command << " | mysql #{username_option} #{password_option} #{database}"
+        command << " | mysql #{username_option} #{host_option} #{password_option} #{database_name}"
 
         run(command)
       end
 
+      def routines_option
+        "--routines"
+      end
+
       def password_option
         "-p'#{password}'" unless password.nil? || password.empty?
       end
 
+      def host_option
+        "-h#{host}"
+      end
+
       def username_option
         "-u#{username}"
       end
 
-      def single_transaction_option(database)
-        '--single-transaction' unless db_has_myisam?(database)
+      def single_transaction_option(database_name)
+        '--single-transaction' unless db_has_myisam?(database_name)
       end
 
-      def db_has_myisam?(database)
-        query, stdout = "SELECT 1 FROM information_schema.tables WHERE table_schema='#{database}' AND engine='MyISAM' LIMIT 1;", StringIO.new
-        spawn(%Q{mysql #{username_option} #{password_option} -N -e"#{query}"}, stdout)
+      def db_has_myisam?(database_name)
+        query, stdout = "SELECT 1 FROM information_schema.tables WHERE table_schema='#{database_name}' AND engine='MyISAM' LIMIT 1;", StringIO.new
+        spawn(%Q{mysql #{username_option} #{password_option} #{host_option} -N -e"#{query}"}, stdout)
         stdout.string.strip == '1'
       end
 
-
+      def suffix
+        /sql\.(gz|gpz)$/
+      end
     end
   end
 end

data/lib/ey_backup/engines/postgresql_engine.rb

@@ -3,13 +3,13 @@ module EY
     class Postgresql < Engine
       register 'postgresql'
 
-      def dump(database, basename)
+      def dump(database_name, basename)
         file = basename + '.pgz'
 
-        command = "PGPASSWORD='#{password}' pg_dump -h localhost --format=c --no-owner --no-privileges -U#{username} #{database}"
+        command = "PGPASSWORD='#{password}' pg_dump -h #{host} --format=c --no-owner --no-privileges -U#{username} #{database_name}"
 
         if gpg?
-          command << " | " << GPGEncryptor.command
+          command << " | " << GPGEncryptor.command_for(key_id)
           file << GPGEncryptor.extension
         end
 
@@ -17,40 +17,46 @@ module EY
 
         run(command)
 
-        Splitter.dump(BackupSet.from(database, file))
+        file
       end
 
-      def load(database, backup)
-        backup = Splitter.load(backup)
-
-        cycle_database(database) if database_exists?(database)
+      def load(database_name, file)
+        if database_exists?(database_name)
+          cycle_database(database_name)
+        else
+          create_database(database_name)
+        end
 
-        command = "cat #{backup.files.first}"
+        command = "cat #{file}"
 
         if gpg?
-          command << " | " << GPGEncryptor.load_command
+          raise "Cannot load a GPG backup"
         end
 
-        command << " | PGPASSWORD='#{password}' pg_restore -h localhost --format=c --ignore-version -U#{username} -d #{database}"
+        command << " | PGPASSWORD='#{password}' pg_restore -h #{host} --format=c --ignore-version -U#{username} -d #{database_name}"
 
         run(command)
       end
 
-      def database_exists?(database)
-        spawn("PGPASSWORD='#{password}' psql -l | grep '#{database}'").success?
+      def database_exists?(database_name)
+        runs?("PGPASSWORD='#{password}' psql -l -h #{host} | grep '#{database_name}'")
+      end
+
+      def drop_database(database_name)
+        spawn("PGPASSWORD='#{password}' dropdb -h #{host} -U#{username} #{database_name}")
       end
 
-      def drop_database(database)
-        spawn("PGPASSWORD='#{password}' dropdb -h localhost -U#{username} #{database}")
+      def create_database(database_name)
+        spawn("PGPASSWORD='#{password}' createdb -U#{username} -h #{host} #{database_name}")
       end
 
-      def create_database(database)
-        spawn("PGPASSWORD='#{password}' createdb -U#{username} -h localhost #{database}")
+      def cycle_database(database_name)
+        drop_database(database_name)
+        create_database(database_name)
       end
 
-      def cycle_database(database)
-        drop_database(database)
-        create_database(database)
+      def suffix
+        /pgz(\.gpz)?$/
       end
     end
   end

data/lib/ey_backup/loader.rb

@@ -3,52 +3,32 @@ module EY
     class Loader < Base
       include Logging
 
-      attr_accessor :database, :index
-
-      def_delegators EY::Backup, :engine
-
-      def self.run(options = {})
-        index, database = options[:index].split(':')
-        new(database, index, options).run
+      def self.run(database, index)
+        new(database, index).run
       end
 
-      def self.download(options = {})
-        index, database = options[:index].split(':')
-        new(database, index, options).download_raw
+      def self.download(database, index)
+        new(database, index).download
       end
 
-      def initialize(database, index, options = {})
-        super(options)
+      def initialize(database, index)
         @database, @index = database, index
       end
 
       def run
+        info "Restoring #{@database.name}"
         backup = download
-        info "Restoring #{database}"
-        engine.load(database, backup)
-        backup.rm!
+        backup.load!
+        backup.remove_joined_file!
       end
 
       def download
-        BackupSet.download(database, index)
-      end
-
-      def download_raw
-        FileUtils.mkdir_p('/mnt/tmp')
-
-        download
-      end
-
-      def base_name
-        "#{base_path}/#{database}.#{timestamp}"
-      end
-
-      def timestamp
-        @timestamp ||= Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')
-      end
+        FileUtils.mkdir_p(EY::Backup.tmp_dir)
 
-      def base_path
-        "/mnt/tmp"
+        backup = BackupSet.download(@database, @index)
+        backup.join!
+        backup.remove_split_files!
+        backup
       end
     end
   end

data/lib/ey_backup/processors/gpg_encryptor.rb

@@ -1,30 +1,13 @@
 module EY
   module Backup
     module GPGEncryptor
-      extend Spawner
-      extend self
-
-      def dump(backup)
-        backup.files = backup.files.map {|file| encrypt(file) }
-        backup
-      end
-
-      def command
-        "gpg --trusted-key #{EY::Backup.config.key_id} --encrypt --recipient #{EY::Backup.config.key_id}"
+      def self.command_for(key_id)
+        "gpg -v --trusted-key #{key_id} --encrypt --recipient #{key_id}"
       end
 
-      def extension
+      def self.extension
         ".gpz"
       end
-
-      def load(backup)
-        backup.files = backup.files.map {|file| decrypt(file) }
-        backup
-      end
-
-      def decrypt(file)
-        fatal "Restoring encrypted backups is not supported."
-      end
     end
   end
 end

data/lib/ey_backup/processors/gzipper.rb

@@ -17,35 +17,6 @@ module EY
       def extension
         ".gz"
       end
-
-      def load(backup)
-        output_files = backup.files.map do |input|
-          if input =~ /\.gz$/
-            output = input.sub(/\.gz$/, '')
-
-            File.open(output, 'w') do |o|
-              File.open(input, 'r') do |i|
-                gz = Zlib::GzipReader.new(i)
-
-                while chunk = gz.read(CHUNK_SIZE)
-                  o << chunk
-                end
-
-                gz.close
-              end
-            end
-
-            FileUtils.rm(input)
-
-            output
-          else
-            input
-          end
-        end
-
-        backup.files = output_files
-        backup
-      end
     end
   end
 end