ey_cloud_server 1.4.5 → 1.4.26

data/lib/ey_backup.rb

@@ -6,74 +6,108 @@ require 'open4'
 require 'zlib'
 require 'ostruct'
 require 'fileutils'
+require 'fog'
 
 module EY
   module Backup
-    extend self
-    attr_accessor :logger, :bucket_minder, :engine, :config
+    class << self
+      attr_accessor :logger
+      attr_accessor :tmp_dir
+    end
 
-    def run(argv = ARGV)
+    def self.run(argv = ARGV)
       options = CLI.run(argv)
-      setup(options)
-      dispatch(options)
+      Runner.new(options).run
     end
 
-    def new_backup(options = {})
-      Dumper.run(options)
-    end
+    class Runner
+      def initialize(options)
+        @options = options
+      end
 
-    def list(options = {})
-      if options[:db].nil? || options[:db].empty?
-        BackupSet.list_all
-      else
-        #list a single database's backups
-        BackupSet.list(options[:db])
+      def run
+        setup
+        dispatch
       end
-    end
 
-    def restore(options = {})
-      Loader.run(options)
-    end
+      # COMMANDS
 
-    def download(options = {})
-      Loader.download(options)
-    end
+      def new_backup
+        Dumper.run(databases, @options[:split_size])
+      end
 
-    def setup(options)
-      setup_config(options)
-      setup_logger(options)
-      setup_minder(options)
-      setup_engine(options)
-    end
+      def list
+        BackupSet.list(databases)
+      end
 
-    def setup_config(options)
-      @config = OpenStruct.new(options)
-    end
+      def restore
+        databases.each do |database|
+          Loader.run(database, @options[:index])
+        end
+      end
 
-    def setup_logger(options)
-      if options[:quiet]
-        @logger ||= Logger.quiet
-      else
-        @logger ||= Logger.new
+      def download
+        databases.each do |database|
+          Loader.download(database, @options[:index])
+        end
       end
-    end
 
-    def setup_minder(options)
-      @bucket        = "ey-backup-#{Digest::SHA1.hexdigest(options[:aws_secret_id])[0..11]}"
-      @bucket_minder = EY::BucketMinder.new(options.merge(:bucket => @bucket))
-    end
+      # END COMMANDS
 
-    def setup_engine(options)
-      @engine = Engine.lookup(options[:engine]).new(options[:dbuser], options[:dbpass])
-    end
+      def databases
+        @databases ||= database_names.map do |name|
+          Database.new(@engine, EY::Backup.tmp_dir, @options[:keep], @backend, @options[:environment], name)
+        end
+      end
 
-    def dispatch(options)
-      send(options[:command], options)
-    end
+      def database_names
+        if @options[:db].nil? || @options[:db].empty?
+          @options[:databases]
+        else
+          [@options[:db]]
+        end
+      end
+
+      def setup
+        setup_logger
+        setup_tmp_dir
+        setup_backend
+        setup_engine
+      end
+
+      def setup_logger
+        if @options[:quiet]
+          EY::Backup.logger ||= Logger.quiet
+        else
+          EY::Backup.logger ||= Logger.new
+        end
+      end
+
+      def setup_tmp_dir
+        if @options[:tmp_dir]
+          EY::Backup.tmp_dir = @options[:tmp_dir]
+        else
+          EY::Backup.tmp_dir = "/mnt/tmp"
+        end
+      end
+
+      def setup_backend
+        @backend = Backend.new(@options[:aws_secret_id], @options[:aws_secret_key], @options[:region], @options[:backup_bucket])
+      end
 
-    def base_path
-      "/mnt/tmp"
+      def setup_engine
+        engine_class = Engine.lookup(@options[:engine])
+        if ! @options.key?(:dbhost) or @options[:dbhost] == nil or @options[:dbhost] == ""
+          @options[:dbhost] = 'localhost'
+        end
+        @engine = engine_class.new(@options[:dbuser], @options[:dbpass], @options[:dbhost], @options[:key_id])
+      end
+
+      def dispatch
+        send(@options[:command])
+      end
     end
+
   end
 end
 
@@ -84,9 +118,11 @@ require lib_dir + '/logger'
 require lib_dir + '/base'
 require lib_dir + '/backup_set'
 require lib_dir + '/cli'
+require lib_dir + '/database'
 require lib_dir + '/dumper'
 require lib_dir + '/engine'
 require lib_dir + '/loader'
+require lib_dir + '/backend'
 
 require lib_dir + '/engines/mysql_engine'
 require lib_dir + '/engines/postgresql_engine'

data/lib/ey_backup/backend.rb

@@ -0,0 +1,34 @@
+require 'fog'
+
+module EY
+  module Backup
+    class Backend
+      include EY::Backup::Logging
+      extend  EY::Backup::Logging
+
+      def initialize(secret_id, secret_key, region, bucket_name)
+        @bucket_minder = EY::BucketMinder.new(secret_id, secret_key, bucket_name, region)
+        @s3            = Fog::Storage.new(:provider => 'AWS', :aws_access_key_id => secret_id, :aws_secret_access_key => secret_key, :region => region)
+      end
+      attr_reader :bucket_minder
+
+      def start_upload(filenames, environment_name, database_name)
+        filenames.each do |filename|
+          begin
+            object_name = "#{environment_name}.#{database_name}/#{File.basename(filename)}"
+            info "Starting upload: #{filename}"
+            @s3.put_object(@bucket_minder.bucket_name, object_name, File.open(filename,'r'))
+            info "Successful upload: #{filename}"
+          rescue => e
+            retries ||= 5
+            retries -= 1
+            raise e if retries == 0
+            warn "retrying upload of #{filename}. Got: #{e.inspect}"
+            retry
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/ey_backup/backup_set.rb

@@ -1,65 +1,56 @@
 module EY
   module Backup
-    class BackupSet < Struct.new(:name, :keys)
+    class BackupSet
       include EY::Backup::Logging
       extend  EY::Backup::Logging
 
-      attr_accessor :files
-
       MULTIPART_EXTENSION = /\.part\d+/
 
-      def self.parse(objects)
-        objects.map {|o| new(o[:name], o[:keys]) }
+      def self.from(database, basename, file)
+        new(database, basename, file, nil)
       end
 
-      def self.from(database, file)
-        backup = new(File.basename(file), [])
-        backup.files = [file]
-        backup
+      def initialize(database, basename, filename, keys)
+        @database = database
+        @basename = basename
+        @filename = filename
+        @keys     = keys
       end
+      attr_reader :basename
 
-      def self.cleanup(database, keep)
-        say "Cleanup for #{database}"
-        backups_for(database)[0...(-keep)].each do |backup|
-          say "deleting: #{backup.name}"
-          backup.delete!
-        end
+      def database_name
+        @database.name
       end
 
-      def self.list_all
-        list('all', backups)
+      def cleanup
+        say "Cleanup for #{database_name}"
+        @database.backups[0...(-@database.keep)].each do |backup|
+          say "Deleting: #{backup.basename}"
+          backup.delete!
+        end
       end
 
-      def self.backups
-        EY::Backup.config.databases.map {|db| backups_for(db) }.flatten
-      end
+      def self.list(databases)
+        names = databases.map do |db|
+          db.name
+        end
+        info "Listing database backups for #{names.join(', ')}"
 
-      def self.list(database, backups = backups_for(database))
-        info "Listing database backups for #{database}"
+        backups = databases.map{ |db| db.backups }.flatten
 
         info "#{backups.size} backup(s) found"
 
-        backups.each_with_index do |db, i|
-          info "#{i}:#{database} #{db.normalized_name}"
+        backups.each_with_index do |backup_set, i|
+          info "#{i}:#{backup_set.database_name} #{backup_set.normalized_name}"
         end
 
         backups
       end
 
-      def self.backups_for(database_name)
-        s3_objects = EY::Backup.bucket_minder.list(:prefix => "#{EY::Backup.config.environment}.#{database_name}", :print => false)
-        parse(s3_objects).sort
-      end
-
-      def self.upload(database, file)
-        EY::Backup.bucket_minder.name = "#{EY::Backup.config.environment}.#{database}/#{File.basename(file)}"
-        EY::Backup.bucket_minder.upload_object(file, false)
-      end
-
       def self.download(database, index)
-        fatal "You didn't specify a database name: e.g. 1:rails_production" if database.empty? || index.empty?
+        fatal "You didn't specify a database name: e.g. 1:rails_production" if database.nil? || index.empty?
 
-        backup = backups_for(database)[index.to_i]
+        backup = database.backups[index.to_i]
 
         fatal "No backup found for database #{database}: requested index: #{index}" unless backup
 
@@ -68,60 +59,76 @@ module EY
         backup
       end
 
-      def initialize(*)
-        super
-
-        @files = []
-      end
-
       def parts
-        [keys.size, files.size].max
+        @keys.size
       end
 
       def normalized_name
-        normalize(name)
+        normalize(@basename)
       end
 
-      def upload!(database)
-        files.each do |file|
-          EY::Backup.bucket_minder.name = "#{EY::Backup.config.environment}.#{database}/#{File.basename(file)}"
-          EY::Backup.bucket_minder.upload_object(file, false)
-          info "successful upload: #{file}"
-        end
+      def load!
+        @database.engine.load(@database.name, @filename)
+      end
+
+      def upload!
+        @database.start_upload(@remote_filenames)
       end
 
       def download!
-        keys.each do |key|
-          info "Downloading #{key}"
-          filename = File.join(EY::Backup.base_path, normalize(key))
-          File.open(filename, 'wb') do |f|
-            EY::Backup.bucket_minder.stream(key) do |chunk|
-              f << chunk
+        @remote_filenames = []
+        @keys.each do |key|
+          info "Downloading #{key} to #{@database.base_path}"
+          remote_filename = File.join(@database.base_path, normalize(key))
+          File.open(remote_filename, 'wb') do |f|
+            @database.bucket_minder.stream(key) do |chunk, remaining_size, total_size|
+              f.write(chunk)
             end
           end
 
-          files << filename
+          @remote_filenames << remote_filename
         end
       end
 
       def delete!
-        keys.each do |key|
-          EY::Backup.bucket_minder.remove_object(key)
+        @keys.each do |key|
+          @database.bucket_minder.remove_object(key)
         end
       end
 
-      def rm!
-        files.each do |file|
-          FileUtils.rm(file)
+      def remove_split_files!
+        (@remote_filenames - [@filename]).each do |filename|
+          FileUtils.rm(filename)
         end
       end
 
+      def remove_joined_file!
+        FileUtils.rm(@filename)
+      end
+
+      def rm!
+        remove_split_files!
+        remove_joined_file!
+      end
+
+      def split!(split_size)
+        @remote_filenames = Splitter.dump(@filename, split_size)
+      end
+
+      def join!
+        @filename = Splitter.load(@remote_filenames)
+      end
+
       def normalize(o)
         o.gsub(/^.*?\//, '')
       end
 
+      def timestamp
+        @basename[/(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})/, 1]
+      end
+
       def <=>(o)
-        self.name <=> o.name
+        self.timestamp <=> o.timestamp
       end
     end
   end

data/lib/ey_backup/base.rb

@@ -5,11 +5,6 @@ module EY
 
       def_delegators :logger, :fatal, :error, :warn, :info, :debug, :say
 
-      def initialize(options = {})
-        @environment = options[:env]
-        @engine_name = options[:engine]
-      end
-
       def logger
         EY::Backup.logger
       end

data/lib/ey_backup/cli.rb

@@ -1,6 +1,6 @@
 module EY
   module Backup
-    module CLI
+    module CLI #rename to option parser and config loader
       extend self
 
       def run(argv)
@@ -27,11 +27,14 @@ module EY
           opts.version = EY::CloudServer::VERSION
 
           opts.banner = "Usage: eybackup [-flag] [argument]"
-          opts.define_head "eybackup: backing up your shit since way back when..."
+          opts.define_head "eybackup: manage dump (mysqldump/pg_dump) style backups of your database."
           opts.separator '*'*80
 
           opts.on("-l", "--list-backup DATABASE", "List mysql backups for DATABASE") do |db|
-            options[:db] = (db || 'all')
+            if db == "all"
+              db = nil
+            end
+            options[:db] = db
             options[:command] = :list
           end
 
@@ -43,18 +46,26 @@ module EY
             options[:config] = config
           end
 
-          opts.on("-d", "--download BACKUP_INDEX", "download the backup specified by index. Run eybackup -l to get the index.") do |index|
+          opts.on("-t", "--tmp_dir TMPDIR", "Use the given directory for temporary storage.") do |tmp_dir|
+            options[:tmp_dir] = tmp_dir
+          end
+
+          opts.on("-d", "--download BACKUP_INDEX", "download the backup specified by index. Run eybackup -l to get the index.") do |index_and_db|
             options[:command] = :download
+            db, index = split_index(index_and_db)
             options[:index] = index
+            options[:db] = db
           end
 
           opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgres.") do |engine|
             options[:engine] = engine
           end
 
-          opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index WARNING! will overwrite the current db with the backup. Run eybackup -l to get the index.") do |index|
+          opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index WARNING! will overwrite the current db with the backup. Run eybackup -l to get the index.") do |index_and_db|
             options[:command] = :restore
+            db, index = split_index(index_and_db)
             options[:index] = index
+            options[:db] = db
           end
 
           opts.on("-k", "--key KEYID", "Public key ID to use for the backup operation") do |key_id|
@@ -75,9 +86,18 @@ module EY
         options
       end
 
+      def split_index(index)
+        index.split(':').reverse
+      end
+
       def config_for(filename)
         if File.exist?(filename)
-          config = YAML::load(File.read(filename))
+          # Make the loaded config have symbols rather than strings
+          config_orig = YAML::load(File.read(filename))
+          config = {}
+          config_orig.each do |key, val|
+            config[key.to_sym] = val
+          end
           config[:environment] ||= config[:env]
           config
         else