dumper 0.0.7 → 0.1.0

data/README.md

@@ -2,7 +2,7 @@
 
 Dumper is a backup management system that offers a whole new way to take daily backups of your databases.
 
-** This app will be launched soon! **
+**This app will be launched soon!**
 
 ## Supported Stacks
 
@@ -25,3 +25,15 @@ then create `config/initializers/dumper.rb` and put the following line.
 ```ruby
 Dumper::Agent.start(:app_key => 'YOUR_APP_KEY')
 ```
+
+or, if you want to conditionally start the agent, pass a block that evaluates to true/false to `#start_if` method.
+
+```ruby
+Dumper::Agent.start_if(:app_key => 'YOUR_APP_KEY') { Rails.env.production? && dumper_enabled_host? }
+```
+
+That's it!
+
+Now, start your server and go to the Dumper site.
+
+You'll find your application is registered and ready to take backups daily.

data/lib/dumper.rb

@@ -10,7 +10,8 @@ module Dumper
   autoload :VERSION,    'dumper/version'
 
   module Database
-    autoload :Base,   'dumper/database/base'
-    autoload :MySQL,  'dumper/database/mysql'
+    autoload :Base,     'dumper/database/base'
+    autoload :MySQL,    'dumper/database/mysql'
+    autoload :MongoDB,  'dumper/database/mongodb'
   end
 end

data/lib/dumper/agent.rb

@@ -31,7 +31,7 @@ module Dumper
     end
 
     def start
-      return unless @app_key and @stack.supported?
+      return unless @stack.supported?
       log "stack: dispatcher = #{@stack.dispatcher}, framework = #{@stack.framework}, rackup = #{@stack.rackup}"
 
       @loop_thread = Thread.new { start_loop }
@@ -42,9 +42,9 @@ module Dumper
       sec = 1
       begin
         sec *= 2
-        log "sleeping #{sec} seconds for agent/register.", :debug
+        log "sleeping #{sec} seconds for agent/register", :debug
         sleep sec
-        json = send_request(api: 'agent/register', json: MultiJson.encode(register_hash))
+        json = api_request('agent/register', :json => MultiJson.dump(register_hash))
       end until json[:status]
 
       return log("agent stopped: #{json.to_s}") if json[:status] == 'error'
@@ -54,7 +54,7 @@ module Dumper
       sleep 1.hour unless @token
 
       loop do
-        json = send_request(api: 'agent/poll', params: { token: @token })
+        json = api_request('agent/poll', :params => { :token => @token })
 
         if json[:status] == 'ok'
           # Promoted or demoted?
@@ -76,22 +76,21 @@ module Dumper
           end
         end
 
-        sleep @token ? 60.seconds : 1.hour
+        sleep [ json[:interval].to_i, 60 ].max
       end
     end
 
     def register_hash
       {
-        # :pid => Process.pid,
-        # :host => Socket.gethostname,
+        :hostname => Socket.gethostname,
         :agent_version => Dumper::VERSION,
         :app_name => @app_name,
         :stack => @stack.to_hash,
       }
     end
 
-    def send_request(options)
-      uri = URI.parse("#{@api_base}/api/#{options[:api]}")
+    def api_request(method_name, options)
+      uri = URI.parse("#{@api_base}/api/#{method_name}")
       http = Net::HTTP.new(uri.host, uri.port)
       if uri.is_a? URI::HTTPS
         http.use_ssl = true
@@ -113,9 +112,9 @@ module Dumper
       response = http.request(request)
       if response.code == '200'
         log response.body, :debug
-        MultiJson.decode(response.body).with_indifferent_access
+        MultiJson.load(response.body).with_indifferent_access
       else
-        log "******** ERROR on api: #{options[:api]}, resp code: #{response.code} ********", :error
+        log "******** ERROR on api: #{method_name}, resp code: #{response.code} ********", :error
         {} # return empty hash
       end
     rescue

data/lib/dumper/database/base.rb

@@ -1,8 +1,30 @@
 module Dumper
   module Database
     class Base
-      def initialize(stack)
+      include Dumper::Utility::ObjectFinder
+
+      attr_accessor :tempfile
+
+      def initialize(stack = nil, options = {})
         @stack = stack
+        @options = options
+      end
+
+      def file_ext
+        self.class::FILE_EXT
+      end
+
+      def dump_tool_path
+        tool = self.class::DUMP_TOOL
+        path = `which #{tool}`.chomp
+        if path.empty?
+          # /usr/local/mysql/bin = OSX binary, /usr/local/bin = homebrew, /usr/bin = linux
+          dir = [ '/usr/local/mysql/bin', '/usr/local/bin', '/usr/bin' ].find do |i|
+            File.exist?("#{i}/#{tool}")
+          end
+          path = "#{dir}/#{tool}" if dir
+        end
+        path
       end
     end
   end

data/lib/dumper/database/mongodb.rb

@@ -0,0 +1,42 @@
+module Dumper
+  module Database
+    class MongoDB < Base
+      DUMP_TOOL = 'mongodump'
+      FILE_EXT = 'tar.gz'
+
+      def command
+        "#{@stack.configs[:mongodb][:dump_tool]} #{connection_options} #{additional_options} && cd #{@options[:tmpdir]} && tar -czf #{@tempfile.path} ."
+      end
+
+      def connection_options
+        [ :database, :host, :port, :username, :password ].map do |option|
+          next if @stack.configs[:mongodb][option].blank?
+          "--#{option}='#{@stack.configs[:mongodb][option]}'".gsub('--database', '--db')
+        end.compact.join(' ')
+      end
+
+      def additional_options
+        "--out='#{@options[:tmpdir]}'"
+      end
+
+      def finalize
+        FileUtils.remove_entry_secure @options[:tmpdir] if File.exist? @options[:tmpdir]
+      end
+
+      def config_for(rails_env=nil)
+        return unless mongo = find_instance_in_object_space(Mongo::DB)
+
+        {
+          :host => mongo.connection.host,
+          :port => mongo.connection.port,
+          :database => mongo.name,
+          :dump_tool => dump_tool_path
+        }.tap do |h|
+          if auth = mongo.connection.auths.first
+            h.update(:username => auth['username'], :password => auth['password'])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dumper/database/mysql.rb

@@ -1,22 +1,39 @@
 module Dumper
   module Database
     class MySQL < Base
+      DUMP_TOOL = 'mysqldump'
+      FILE_EXT = 'sql.gz'
+
       def command
-        "#{@stack.dump_tool(:mysqldump)} #{connection_options} #{additional_options} #{@stack.activerecord_config['database']} | gzip"
+        "#{@stack.configs[:mysql][:dump_tool]} #{connection_options} #{additional_options} #{@stack.configs[:mysql][:database]} | gzip > #{@tempfile.path}"
       end
 
-      protected
-
       def connection_options
         [ :host, :port, :username, :password ].map do |option|
-          next if @stack.activerecord_config[option.to_s].blank?
-          "--#{option}='#{@stack.activerecord_config[option.to_s]}'".gsub('--username', '--user')
+          next if @stack.configs[:mysql][option].blank?
+          "--#{option}='#{@stack.configs[:mysql][option]}'".gsub('--username', '--user')
         end.compact.join(' ')
       end
 
       def additional_options
         '--single-transaction'
       end
+
+      def config_for(rails_env=nil)
+        return unless defined?(ActiveRecord::Base) &&
+          ActiveRecord::Base.configurations &&
+          (config = ActiveRecord::Base.configurations[rails_env]) &&
+          %w(mysql mysql2).include?(config['adapter'])
+
+        {
+          :host => config['host'],
+          :port => config['port'],
+          :username => config['username'],
+          :password => config['password'],
+          :database => config['database'],
+          :dump_tool => dump_tool_path
+        }
+      end
     end
   end
 end

data/lib/dumper/job.rb

@@ -5,6 +5,8 @@ module Dumper
     include POSIX::Spawn
     include Dumper::Utility::LoggingMethods
 
+    MAX_FILESIZE = 4.gigabytes
+
     def initialize(agent, job)
       @agent = agent
       @stack = agent.stack
@@ -18,26 +20,32 @@ module Dumper
     ensure
       log_last_error if $!
       log 'exiting...'
-      exit
+      exit!(true) # Do not use exit or abort to skip at_exit execution, or pid could get deleted on thin
     end
 
     def perform(server)
-      # Prepare
-      json = @agent.send_request(api: 'backup/prepare', params: { server_id: server[:id], manual: server[:manual].to_s })
-      return unless json[:status] == 'ok'
-
+      # Initialize database
       case server[:type]
       when 'mysql'
         @database = Dumper::Database::MySQL.new(@stack)
+      when 'mongodb'
+        @database = Dumper::Database::MongoDB.new(@stack, :tmpdir => Dir.mktmpdir)
       else
-        abort 'invalid server type!' # TBD
+        log "invalid server type: #{server[:type]}"
+        exit!
       end
+
+      # Prepare
+      json = @agent.api_request('backup/prepare', :params => { :server_id => server[:id], :manual => server[:manual].to_s, :ext => @database.file_ext })
+      return unless json[:status] == 'ok'
+
       backup_id = json[:backup][:id]
       filename = json[:backup][:filename]
 
       # Dump
       start_at = Time.now
       tempfile = ruby19? ? Tempfile.new(filename, encoding: 'ascii-8bit') : Tempfile.new(filename)
+      @database.tempfile = tempfile
       log 'starting backup...'
       log "tempfile = #{tempfile.path}"
       log "command = #{@database.command}"
@@ -45,36 +53,37 @@ module Dumper
       begin
         pid, stdin, stdout, stderr = popen4(@database.command)
         stdin.close
-        # Reuse buffer: http://www.ruby-forum.com/topic/134164
-        buffer_size = 1.megabytes
-        buffer = "\x00" * buffer_size # fixed-size malloc optimization
-        while stdout.read(buffer_size, buffer)
-          tempfile.write buffer
-          if tempfile.size > Backup::MAX_FILESIZE
-            raise 'Max filesize exceeded.'
-          end
-        end
+        # # Reuse buffer: http://www.ruby-forum.com/topic/134164
+        # buffer_size = 1.megabytes
+        # buffer = "\x00" * buffer_size # fixed-size malloc optimization
+        # while stdout.read(buffer_size, buffer)
+        #   tempfile.write buffer
+        #   if tempfile.size > MAX_FILESIZE
+        #     raise 'Max filesize exceeded.'
+        #   end
+        # end
+        # tempfile.flush
       rescue
         Process.kill(:INT, pid) rescue SystemCallError
-        @agent.send_request(api: 'backup/fail', params: { backup_id: backup_id, code: 'dump_error', message: $!.to_s })
-        abort
+        @database.finalize if @database.respond_to?(:finalize)
+        @agent.api_request('backup/fail', :params => { :backup_id => backup_id, :code => 'dump_error', :message => $!.to_s })
+        exit!
       ensure
         [stdin, stdout, stderr].each{|io| io.close unless io.closed? }
         Process.waitpid(pid)
       end
 
-      tempfile.flush
-
       dump_duration = Time.now - start_at
       log "dump_duration = #{dump_duration}"
 
       upload_to_s3(json[:url], json[:fields], tempfile.path, filename)
 
-      json = @agent.send_request(api: 'backup/commit', params: { backup_id: backup_id, dump_duration: dump_duration.to_i })
+      json = @agent.api_request('backup/commit', :params => { :backup_id => backup_id, :dump_duration => dump_duration.to_i })
     rescue
       log_last_error
     ensure
       tempfile.close(true)
+      @database.finalize if @database.respond_to?(:finalize)
     end
 
     # Upload

data/lib/dumper/stack.rb

@@ -1,10 +1,17 @@
 module Dumper
   class Stack
-    DUMP_TOOLS = {}
+    include Dumper::Utility::ObjectFinder
 
-    attr_accessor :rails_env, :dispatcher, :framework, :rackup, :activerecord_config
+    DATABASES = {
+      :mysql => Dumper::Database::MySQL,
+      :mongodb => Dumper::Database::MongoDB,
+    }
+
+    attr_accessor :rails_env, :dispatcher, :framework, :rackup, :configs
 
     def initialize
+      @configs = {}
+
       # Rackup?
       @rackup = find_instance_in_object_space(Rack::Server)
 
@@ -14,7 +21,11 @@ module Dumper
         @rails_env = Rails.env.to_s
         @rails_version = Rails::VERSION::STRING
         @is_supported_rails_version = (::Rails::VERSION::MAJOR >= 3)
-        @activerecord_config = ActiveRecord::Base.configurations[@rails_env]
+        DATABASES.each do |key, klass|
+          next unless config = klass.new.config_for(@rails_env)
+          @configs[key] = config
+        end
+
       else
         @framework = :ruby
       end
@@ -31,29 +42,13 @@ module Dumper
         rails_env: @rails_env,
         rails_version: @rails_version,
         dispatcher: @dispatcher,
-        activerecord_config: @activerecord_config.reject{|k,v| k == 'password' },
-        dump_tools: DUMP_TOOLS,
+        configs: Hash[@configs.map{|k, config| [ k, config.reject{|k,v| k == :password } ] }]
       }
     end
 
     # Compatibility
     def supported?
-      @is_supported_rails_version && @dispatcher && mysql?
-    end
-
-    # Database
-    def mysql?
-      dump_tool(:mysqldump) # Just assign DUMP_TOOLS
-      %w(mysql mysql2).include?(@activerecord_config['adapter'])
-    end
-
-    # Dump Tool
-    def dump_tool(name)
-      DUMP_TOOLS[name] ||= `which #{name}`.chomp
-    end
-
-    def dump_tool_installed?(name)
-      !dump_tool(name).empty?
+      @is_supported_rails_version && @dispatcher && !@configs.empty?
     end
 
     # Dispatcher
@@ -66,8 +61,8 @@ module Dumper
     end
 
     def thin?
-      # defined?(::Thin::Server) && find_instance_in_object_space(Thin::Server)
-      @rackup and @rackup.server.to_s.demodulize == 'Thin'
+      defined?(::Thin::Server) && find_instance_in_object_space(::Thin::Server) ||
+        (@rackup && @rackup.server.to_s.demodulize == 'Thin')
     end
 
     def mongrel?
@@ -79,9 +74,5 @@ module Dumper
       # defined?(::WEBrick::VERSION)
       @rackup and @rackup.server.to_s.demodulize == 'WEBrick'
     end
-
-    def find_instance_in_object_space(klass)
-      ObjectSpace.each_object(klass).first
-    end
   end
 end

data/lib/dumper/utility.rb

@@ -48,7 +48,7 @@ module Dumper
       class SlimFormatter < Logger::Formatter
         # This method is invoked when a log event occurs
         def call(severity, time, progname, msg)
-          "[%s] %5s : %s\n" % [format_datetime(time), severity, msg2str(msg)]
+          "[%s (%d)] %5s : %s\n" % [format_datetime(time), $$, severity, msg2str(msg)]
         end
       end
     end
@@ -72,5 +72,11 @@ module Dumper
         log [ $!.class.name, $!.to_s ].join(', ')
       end
     end
+
+    module ObjectFinder
+      def find_instance_in_object_space(klass)
+        ObjectSpace.each_object(klass).first
+      end
+    end
   end
 end

data/lib/dumper/version.rb

@@ -1,3 +1,3 @@
 module Dumper
-  VERSION = '0.0.7'
+  VERSION = '0.1.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dumper
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-30 00:00:00.000000000 Z
+date: 2012-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -112,6 +112,7 @@ files:
 - lib/dumper/agent.rb
 - lib/dumper/cli.rb
 - lib/dumper/database/base.rb
+- lib/dumper/database/mongodb.rb
 - lib/dumper/database/mysql.rb
 - lib/dumper/dependency.rb
 - lib/dumper/job.rb