ey_cloud_server 1.3.1 → 1.4.5.pre

data/lib/ey_backup/dumper.rb

@@ -0,0 +1,47 @@
+module EY
+  module Backup
+    class Dumper < Base
+      attr_accessor :database
+
+      def_delegators EY::Backup, :engine, :minder, :base_path
+
+      def self.run(options = {})
+        if database = options[:db]
+          new(database, options).run
+        else
+          options[:databases].each do |database|
+            new(database, options).run
+          end
+        end
+      end
+
+      def initialize(database, options = {})
+        super(options)
+        @database = database
+      end
+
+      def run
+        info "doing database: #{database}"
+
+        FileUtils.mkdir_p('/mnt/tmp')
+
+        backup = engine.dump(database, base_name)
+
+        backup.upload!(database)
+
+        info "Successful backup: #{database}.#{backup.normalized_name}"
+        BackupSet.cleanup(database, EY::Backup.config.keep)
+      end
+
+
+      def base_name
+        "#{base_path}/#{database}.#{timestamp}"
+      end
+
+      def timestamp
+        @timestamp ||= Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')
+      end
+
+    end
+  end
+end

data/lib/ey_backup/engine.rb

@@ -0,0 +1,51 @@
+module EY
+  module Backup
+    class Engine < Base
+      include Spawner
+
+      attr_accessor :username, :password
+
+      def self.label
+        @label
+      end
+
+      def self.register(label)
+        @label = label
+      end
+
+      def self.descendants
+        @descendants ||= []
+      end
+
+      def self.inherited(descendant)
+        descendants << descendant
+      end
+
+      def self.lookup(label)
+        descendants.detect {|d| d.label == label } ||
+          EY::Backup.logger.fatal("Unknown database engine: #{label}")
+      end
+
+      def initialize(username, password)
+        @username, @password = username, password
+      end
+
+      def post_dump_process(backup)
+        processors.inject(backup) do |backup, processor|
+          processor.dump(backup)
+        end
+      end
+
+      def pre_load_process(backup)
+        processors.reverse.inject(backup) do |backup, processor|
+          processor.load(backup)
+        end
+      end
+
+      def gpg?
+        not EY::Backup.config.key_id.nil? &&
+          EY::Backup.config.key_id.blank?
+      end
+    end
+  end
+end

data/lib/ey_backup/engines/mysql_engine.rb

@@ -0,0 +1,63 @@
+module EY
+  module Backup
+    class MysqlEngine < Engine
+      register 'mysql'
+
+      def dump(database, basename)
+        file = basename + '.sql'
+
+        command = "mysqldump #{username_option} #{password_option} #{single_transaction_option(database)} #{database}"
+
+        if gpg?
+          command << " | " << GPGEncryptor.command
+          file << GPGEncryptor.extension
+        else
+          command << " | " << GZipper.gzip
+          file << GZipper.extension
+        end
+
+        command << " > #{file}"
+
+        run(command)
+
+        Splitter.dump(BackupSet.from(database, file))
+      end
+
+      def load(database, backup)
+        backup = Splitter.load(backup)
+
+        command = "cat #{backup.files.first}"
+
+        if gpg?
+          command << " | " << GPGEncryptor.load_command
+        else
+          command << " | " << GZipper.gunzip
+        end
+
+        command << " | mysql #{username_option} #{password_option} #{database}"
+
+        run(command)
+      end
+
+      def password_option
+        "-p'#{password}'" unless password.nil? || password.empty?
+      end
+
+      def username_option
+        "-u#{username}"
+      end
+
+      def single_transaction_option(database)
+        '--single-transaction' unless db_has_myisam?(database)
+      end
+
+      def db_has_myisam?(database)
+        query, stdout = "SELECT 1 FROM information_schema.tables WHERE table_schema='#{database}' AND engine='MyISAM' LIMIT 1;", StringIO.new
+        spawn(%Q{mysql #{username_option} #{password_option} -N -e"#{query}"}, stdout)
+        stdout.string.strip == '1'
+      end
+
+
+    end
+  end
+end

data/lib/ey_backup/engines/postgresql_engine.rb

@@ -0,0 +1,57 @@
+module EY
+  module Backup
+    class Postgresql < Engine
+      register 'postgresql'
+
+      def dump(database, basename)
+        file = basename + '.pgz'
+
+        command = "PGPASSWORD='#{password}' pg_dump -h localhost --format=c --no-owner --no-privileges -U#{username} #{database}"
+
+        if gpg?
+          command << " | " << GPGEncryptor.command
+          file << GPGEncryptor.extension
+        end
+
+        command << " > #{file}"
+
+        run(command)
+
+        Splitter.dump(BackupSet.from(database, file))
+      end
+
+      def load(database, backup)
+        backup = Splitter.load(backup)
+
+        cycle_database(database) if database_exists?(database)
+
+        command = "cat #{backup.files.first}"
+
+        if gpg?
+          command << " | " << GPGEncryptor.load_command
+        end
+
+        command << " | PGPASSWORD='#{password}' pg_restore -h localhost --format=c --ignore-version -U#{username} -d #{database}"
+
+        run(command)
+      end
+
+      def database_exists?(database)
+        spawn("PGPASSWORD='#{password}' psql -l | grep '#{database}'").success?
+      end
+
+      def drop_database(database)
+        spawn("PGPASSWORD='#{password}' dropdb -h localhost -U#{username} #{database}")
+      end
+
+      def create_database(database)
+        spawn("PGPASSWORD='#{password}' createdb -U#{username} -h localhost #{database}")
+      end
+
+      def cycle_database(database)
+        drop_database(database)
+        create_database(database)
+      end
+    end
+  end
+end

data/lib/ey_backup/loader.rb

@@ -0,0 +1,55 @@
+module EY
+  module Backup
+    class Loader < Base
+      include Logging
+
+      attr_accessor :database, :index
+
+      def_delegators EY::Backup, :engine
+
+      def self.run(options = {})
+        index, database = options[:index].split(':')
+        new(database, index, options).run
+      end
+
+      def self.download(options = {})
+        index, database = options[:index].split(':')
+        new(database, index, options).download_raw
+      end
+
+      def initialize(database, index, options = {})
+        super(options)
+        @database, @index = database, index
+      end
+
+      def run
+        backup = download
+        info "Restoring #{database}"
+        engine.load(database, backup)
+        backup.rm!
+      end
+
+      def download
+        BackupSet.download(database, index)
+      end
+
+      def download_raw
+        FileUtils.mkdir_p('/mnt/tmp')
+
+        download
+      end
+
+      def base_name
+        "#{base_path}/#{database}.#{timestamp}"
+      end
+
+      def timestamp
+        @timestamp ||= Time.now.strftime("%Y-%m-%dT%H:%M:%S").gsub(/:/, '-')
+      end
+
+      def base_path
+        "/mnt/tmp"
+      end
+    end
+  end
+end

data/lib/ey_backup/logger.rb

@@ -0,0 +1,36 @@
+module EY
+  module Backup
+    class Logger
+      extend Forwardable
+
+      attr_accessor :stdout, :stderr
+
+      alias_method  :fatal, :abort
+      public :fatal
+
+      def_delegator :stderr, :puts, :error
+      def_delegator :stderr, :puts, :warn
+      def_delegator :stdout, :puts, :info
+      def_delegator :stdout, :puts, :puts
+
+      def self.quiet
+        new(StringIO.new)
+      end
+
+      def initialize(stdout = $stdout, stderr = $stderr)
+        @stdout, @stderr = stdout, stderr
+      end
+
+      def say(msg, newline = true)
+        newline ? info(msg) : stdout.print(msg)
+      end
+    end
+
+    module Logging
+      extend Forwardable
+
+      def_delegator EY::Backup, :logger
+      def_delegators :logger, :fatal, :error, :warn, :info, :puts, :say
+    end
+  end
+end

data/lib/ey_backup/processors/gpg_encryptor.rb

@@ -0,0 +1,30 @@
+module EY
+  module Backup
+    module GPGEncryptor
+      extend Spawner
+      extend self
+
+      def dump(backup)
+        backup.files = backup.files.map {|file| encrypt(file) }
+        backup
+      end
+
+      def command
+        "gpg --trusted-key #{EY::Backup.config.key_id} --encrypt --recipient #{EY::Backup.config.key_id}"
+      end
+
+      def extension
+        ".gpz"
+      end
+
+      def load(backup)
+        backup.files = backup.files.map {|file| decrypt(file) }
+        backup
+      end
+
+      def decrypt(file)
+        fatal "Restoring encrypted backups is not supported."
+      end
+    end
+  end
+end

data/lib/ey_backup/processors/gzipper.rb

@@ -0,0 +1,53 @@
+module EY
+  module Backup
+    module GZipper
+      extend Spawner
+      extend self
+
+      CHUNK_SIZE = 1024 * 64
+
+      def gzip
+        "gzip -c"
+      end
+
+      def gunzip
+        "gunzip -c"
+      end
+
+      def extension
+        ".gz"
+      end
+
+      def load(backup)
+        output_files = backup.files.map do |input|
+          if input =~ /\.gz$/
+            output = input.sub(/\.gz$/, '')
+
+            File.open(output, 'w') do |o|
+              File.open(input, 'r') do |i|
+                gz = Zlib::GzipReader.new(i)
+
+                while chunk = gz.read(CHUNK_SIZE)
+                  o << chunk
+                end
+
+                gz.close
+              end
+            end
+
+            FileUtils.rm(input)
+
+            output
+          else
+            input
+          end
+        end
+
+        backup.files = output_files
+        backup
+      end
+    end
+  end
+end
+
+

data/lib/ey_backup/processors/splitter.rb

@@ -0,0 +1,96 @@
+module EY
+  module Backup
+    module Splitter
+      extend Spawner
+      extend Logging
+      extend self
+
+      CHUNK_SIZE = 1024 * 64
+      MAX_FILE_SIZE = (4.5*1024*1024*1024).to_i #4.5GB
+
+      def load(backup)
+        if backup.files.size > 1
+          backup.files = join(backup.files)
+        end
+
+        backup
+      end
+
+      def dump(command, files)
+        if files.size != 1
+          fatal "Can't split multiple files."
+        end
+
+      end
+
+      def dump(backup)
+        output_files = backup.files.map do |file|
+          if File.size(file) >= split_size
+            split(file)
+          else
+            file
+          end
+        end.flatten
+
+        backup.files = output_files
+        backup
+      end
+
+      def join(input_files)
+        filename = input_files.first.sub(/\.part\d+$/, '')
+
+        File.open(filename, 'w') do |output|
+          sort(input_files).each do |input|
+            File.open(input, 'r') do |i|
+              while chunk = i.read(CHUNK_SIZE)
+                output << chunk
+              end
+            end
+
+            FileUtils.rm(input)
+          end
+        end
+
+        [filename]
+      end
+
+      def sort(input_files)
+        input_files.sort {|a,b| part_number(a) <=> part_number(b) }
+      end
+
+      def part_number(file)
+        file[/\.part(\d+)/, 1].to_i
+      end
+
+      def split(file)
+        total_size, part = 0, 0
+        files = []
+        File.open(file, 'r') do |i|
+          until total_size == File.size(file)
+            part += 1
+            part_size = 0
+            filename = "#{file}.part#{part}"
+            File.open(filename, 'w') do |o|
+              while part_size < split_size && (chunk = i.read([split_size - part_size, CHUNK_SIZE].min))
+                part_size += chunk.size
+                o << chunk
+              end
+            end
+
+            total_size += part_size
+
+            files << filename
+          end
+        end
+
+        FileUtils.rm(file)
+
+        files
+      end
+
+      def split_size
+        EY::Backup.config.split_size || MAX_FILE_SIZE
+      end
+    end
+  end
+end