dumper 0.1.1 → 0.2.0

data/README.md

@@ -9,8 +9,9 @@ Dumper is a backup management system that offers a whole new way to take daily b
 * Ruby 1.8.7 , Ruby 1.9.2 or later
 * Rails 3.0 or later
 * MySQL with ActiveRecord
+* MongoDB with mongo gem (including Mongoid 2.x)
 
-Support for PostgreSQL, MongoDB and Redis are coming soon.
+Support for PostgreSQL and Redis are coming soon.
 
 ## Installation
 

data/lib/dumper.rb

@@ -13,5 +13,6 @@ module Dumper
     autoload :Base,     'dumper/database/base'
     autoload :MySQL,    'dumper/database/mysql'
     autoload :MongoDB,  'dumper/database/mongodb'
+    autoload :Redis,    'dumper/database/redis'
   end
 end

data/lib/dumper/agent.rb

@@ -20,7 +20,7 @@ module Dumper
     end
 
     def initialize(options = {})
-      log '**** Dumper requires :app_key! ****' if options[:app_key].blank?
+      log 'app_key is missing' if options[:app_key].blank?
 
       @stack = Dumper::Stack.new
       @api_base = options[:api_base] || 'http://dumper.io'
@@ -51,7 +51,7 @@ module Dumper
 
       @token = json[:token]
       log "agent started as #{@token ? 'primary' : 'secondary'}"
-      sleep 1.hour unless @token
+      sleep 1.hour + rand(10) unless @token
 
       loop do
         json = api_request('agent/poll', :params => { :token => @token })

data/lib/dumper/database/base.rb

@@ -3,17 +3,20 @@ module Dumper
     class Base
       include Dumper::Utility::ObjectFinder
 
-      attr_accessor :tempfile
+      attr_accessor :tmpdir, :filename
 
-      def initialize(stack = nil, options = {})
+      def initialize(stack = nil)
         @stack = stack
-        @options = options
       end
 
       def file_ext
         self.class::FILE_EXT
       end
 
+      def dump_path
+        "#{tmpdir}/#{filename}"
+      end
+
       def dump_tool_path
         tool = self.class::DUMP_TOOL
         path = `which #{tool}`.chomp
@@ -26,6 +29,10 @@ module Dumper
         end
         path
       end
+
+      def finalize
+        FileUtils.remove_entry_secure tmpdir if File.exist? tmpdir
+      end
     end
   end
 end

data/lib/dumper/database/mongodb.rb

@@ -5,7 +5,7 @@ module Dumper
       FILE_EXT = 'tar.gz'
 
       def command
-        "#{@stack.configs[:mongodb][:dump_tool]} #{connection_options} #{additional_options} && cd #{@options[:tmpdir]} && tar -czf #{@tempfile.path} ."
+        "cd #{tmpdir} && #{dump_tool_path} #{connection_options} #{additional_options} && tar -czf #{filename} ."
       end
 
       def connection_options
@@ -16,16 +16,12 @@ module Dumper
       end
 
       def additional_options
-        "--out='#{@options[:tmpdir]}'"
-      end
-
-      def finalize
-        FileUtils.remove_entry_secure @options[:tmpdir] if File.exist? @options[:tmpdir]
+        "--out='#{tmpdir}'"
       end
 
       def config_for(rails_env=nil)
         return unless defined?(Mongo::DB) &&
-          mongo = find_instance_in_object_space(Mongo::DB)
+          (mongo = find_instance_in_object_space(Mongo::DB))
 
         {
           :host => mongo.connection.host,

data/lib/dumper/database/mysql.rb

@@ -5,7 +5,7 @@ module Dumper
       FILE_EXT = 'sql.gz'
 
       def command
-        "#{@stack.configs[:mysql][:dump_tool]} #{connection_options} #{additional_options} #{@stack.configs[:mysql][:database]} | gzip > #{@tempfile.path}"
+        "cd #{tmpdir} && #{dump_tool_path} #{connection_options} #{additional_options} #{@stack.configs[:mysql][:database]} | gzip > #{filename}"
       end
 
       def connection_options

data/lib/dumper/database/redis.rb

@@ -0,0 +1,37 @@
+module Dumper
+  module Database
+    class Redis < Base
+      DUMP_TOOL = 'redis-cli'
+      FILE_EXT = 'rdb.gz'
+
+      def command
+        uncompressed = filename.sub('.gz','')
+        "cd #{tmpdir} && cp #{@stack.configs[:redis][:dbpath]} #{uncompressed} && gzip #{uncompressed}"
+      end
+
+      def config_for(rails_env=nil)
+        return unless defined?(::Redis) &&
+          (main_thread_redis = find_instance_in_object_space(::Redis))
+
+        client = main_thread_redis.client
+
+        # New connection for the agent thread
+        redis = ::Redis.new(:host => client.host, :port => client.port, :password => client.password, :db => client.db)
+        dir         = redis.config(:get, :dir)['dir']
+        dbfilename  = redis.config(:get, :dbfilename)['dbfilename']
+        dbpath = "#{dir}/#{dbfilename}"
+
+        return unless File.exist?(dbpath) # Redis must run on the back up node
+
+        {
+          :host => client.host,
+          :port => client.port,
+          :password => client.password,
+          :database => client.db,
+          :dump_tool => dump_tool_path,
+          :dbpath => dbpath
+        }
+      end
+    end
+  end
+end

data/lib/dumper/job.rb

@@ -29,45 +29,35 @@ module Dumper
       when 'mysql'
         @database = Dumper::Database::MySQL.new(@stack)
       when 'mongodb'
-        @database = Dumper::Database::MongoDB.new(@stack, :tmpdir => Dir.mktmpdir)
+        @database = Dumper::Database::MongoDB.new(@stack)
+      when 'redis'
+        @database = Dumper::Database::Redis.new(@stack)
       else
-        log "invalid server type: #{server[:type]}"
-        exit!
+        abort_with "invalid server type: #{server[:type]}"
       end
 
       # Prepare
       json = @agent.api_request('backup/prepare', :params => { :server_id => server[:id], :manual => server[:manual].to_s, :ext => @database.file_ext })
-      return unless json[:status] == 'ok'
+      abort_with('backup/prepare failed') unless json[:status] == 'ok'
 
       backup_id = json[:backup][:id]
-      filename = json[:backup][:filename]
 
       # Dump
       start_at = Time.now
-      tempfile = ruby19? ? Tempfile.new(filename, encoding: 'ascii-8bit') : Tempfile.new(filename)
-      @database.tempfile = tempfile
+      @database.tmpdir = Dir.mktmpdir
+      @database.filename = json[:backup][:filename]
       log 'starting backup...'
-      log "tempfile = #{tempfile.path}"
+      log "tmpdir = #{@database.tmpdir}, filename = #{@database.filename}"
       log "command = #{@database.command}"
 
       begin
         pid, stdin, stdout, stderr = popen4(@database.command)
         stdin.close
-        # # Reuse buffer: http://www.ruby-forum.com/topic/134164
-        # buffer_size = 1.megabytes
-        # buffer = "\x00" * buffer_size # fixed-size malloc optimization
-        # while stdout.read(buffer_size, buffer)
-        #   tempfile.write buffer
-        #   if tempfile.size > MAX_FILESIZE
-        #     raise 'Max filesize exceeded.'
-        #   end
-        # end
-        # tempfile.flush
       rescue
         Process.kill(:INT, pid) rescue SystemCallError
-        @database.finalize if @database.respond_to?(:finalize)
+        @database.finalize
         @agent.api_request('backup/fail', :params => { :backup_id => backup_id, :code => 'dump_error', :message => $!.to_s })
-        exit!
+        abort_with("dump error: #{$!}")
       ensure
         [stdin, stdout, stderr].each{|io| io.close unless io.closed? }
         Process.waitpid(pid)
@@ -75,21 +65,21 @@ module Dumper
 
       dump_duration = Time.now - start_at
       log "dump_duration = #{dump_duration}"
+      abort_with('max filesize exceeded') if File.size(@database.dump_path) > MAX_FILESIZE
 
-      upload_to_s3(json[:url], json[:fields], tempfile.path, filename)
+      upload_to_s3(json[:url], json[:fields])
 
       json = @agent.api_request('backup/commit', :params => { :backup_id => backup_id, :dump_duration => dump_duration.to_i })
     rescue
       log_last_error
     ensure
-      tempfile.close(true)
-      @database.finalize if @database.respond_to?(:finalize)
+      @database.finalize
     end
 
     # Upload
-    def upload_to_s3(url, fields, local_file, remote_file)
+    def upload_to_s3(url, fields)
       require 'net/http/post/multipart'
-      fields['file'] = UploadIO.new(local_file, 'application/octet-stream', remote_file)
+      fields['file'] = UploadIO.new(@database.dump_path, 'application/octet-stream', @database.filename)
       uri = URI.parse(url)
       request = Net::HTTP::Post::Multipart.new uri.path, fields
       http = Net::HTTP.new(uri.host, uri.port)
@@ -104,9 +94,10 @@ module Dumper
       log_last_error
     end
 
-    # Helper
-    def ruby19?
-      RUBY_VERSION >= '1.9.0'
+    def abort_with(text)
+      log text
+      @database.try(:finalize)
+      exit!
     end
   end
 end

data/lib/dumper/stack.rb

@@ -5,6 +5,7 @@ module Dumper
     DATABASES = {
       :mysql => Dumper::Database::MySQL,
       :mongodb => Dumper::Database::MongoDB,
+      :redis => Dumper::Database::Redis,
     }
 
     attr_accessor :rails_env, :dispatcher, :framework, :rackup, :configs

data/lib/dumper/version.rb

@@ -1,3 +1,3 @@
 module Dumper
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dumper
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-31 00:00:00.000000000 Z
+date: 2012-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -114,6 +114,7 @@ files:
 - lib/dumper/database/base.rb
 - lib/dumper/database/mongodb.rb
 - lib/dumper/database/mysql.rb
+- lib/dumper/database/redis.rb
 - lib/dumper/dependency.rb
 - lib/dumper/job.rb
 - lib/dumper/stack.rb