dumper 0.0.3 → 0.0.4

data/.gitignore

@@ -1,17 +1,2 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
 Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
 pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp

data/.rspec

@@ -0,0 +1 @@
+--color

data/README.md

@@ -1,19 +1,27 @@
-# Dumper
+# Dumper Agent for Rails
 
-Utility that checks the status of a database.
+Dumper is a backup management system that offers a whole new way to take daily backups of your databases.
 
-## Installation
+** This app will be launched soon! **
+
+## Supported Stacks
+
+* Ruby 1.8.7 , Ruby 1.9.2 or later
+* Rails 3.0 or later
+* MySQL with ActiveRecord
 
-    $ gem install dumper
+Support for PostgreSQL, MongoDB and Redis are coming soon.
+
+## Installation
 
-## Usage
+Add the following line to your Rails project Gemfile:
 
-TODO: Write usage instructions here
+```ruby
+gem 'dumper'
+```
 
-## Contributing
+then create `config/initializers/dumper.rb` and put the following line.
 
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+```ruby
+Dumper::Agent.start(:app_key => 'YOUR_APP_KEY')
+```

data/Rakefile

@@ -1,2 +1,7 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
+
+# RSpec
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/dumper.gemspec

@@ -2,19 +2,23 @@
 require File.expand_path('../lib/dumper/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Kenn Ejima"]
-  gem.email         = ["kenn.ejima@gmail.com"]
-  gem.description   = %q{Utility that checks the status of a database}
-  gem.summary       = %q{Utility that checks the status of a database}
-  gem.homepage      = ""
+  gem.authors       = ['Kenn Ejima']
+  gem.email         = ['kenn.ejima@gmail.com']
+  gem.description   = 'Dumper is a backup management system that offers a whole new way to take daily backups of your databases. (coming soon!)'
+  gem.summary       = 'The Dumper Agent for Rails (coming soon!)'
+  gem.homepage      = 'https://github.com/kenn/dumper'
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.name          = "dumper"
-  gem.require_paths = ["lib"]
+  gem.name          = 'dumper'
+  gem.require_paths = ['lib']
   gem.version       = Dumper::VERSION
 
-  gem.add_runtime_dependency "thor"
-  gem.add_runtime_dependency "rainbow"
+  gem.add_runtime_dependency 'multi_json', '~> 1.0'
+  gem.add_runtime_dependency 'posix-spawn', '~> 0.3.6'
+  gem.add_development_dependency 'rspec'
+
+  # For Travis
+  gem.add_development_dependency 'rake'
 end

data/lib/dumper.rb

@@ -1,9 +1,16 @@
-require 'thor'
-require 'rainbow'
 require 'socket'
 
 module Dumper
-  autoload :Cli,      'dumper/cli'
-  autoload :Utility,  'dumper/utility'
-  autoload :VERSION,  'dumper/version'
+  autoload :Agent,      'dumper/agent'
+  autoload :Cli,        'dumper/cli'
+  autoload :Dependency, 'dumper/dependency'
+  autoload :Job,        'dumper/job'
+  autoload :Stack,      'dumper/stack'
+  autoload :Utility,    'dumper/utility'
+  autoload :VERSION,    'dumper/version'
+
+  module Database
+    autoload :Base,   'dumper/database/base'
+    autoload :MySQL,  'dumper/database/mysql'
+  end
 end

data/lib/dumper/agent.rb

@@ -0,0 +1,112 @@
+require 'timeout'
+require 'net/http'
+
+module Dumper
+  class Agent
+    include Dumper::Utility::LoggingMethods
+
+    API_VERSION = 1
+
+    attr_reader :stack
+
+    class << self
+      def start(options = {})
+        new(options).start
+      end
+
+      def start_if(options = {})
+        start(options) if yield
+      end
+    end
+
+    def initialize(options = {})
+      log '**** Dumper requires :app_key! ****' if options[:app_key].blank?
+
+      @stack = Dumper::Stack.new
+      @api_base = options[:api_base] || 'http://dumper.io'
+      @app_key = options[:app_key]
+      @app_env = @stack.rails_env
+      @app_name = ObjectSpace.each_object(Rails::Application).first.class.name.split("::").first
+    end
+
+    def start
+      return unless @app_key and @stack.supported?
+      log "stack: dispatcher = #{@stack.dispatcher}, framework = #{@stack.framework}, rackup = #{@stack.rackup}"
+
+      @loop_thread = Thread.new { start_loop }
+      @loop_thread[:name] = 'Loop Thread'
+    end
+
+    def start_loop
+      sleep 3
+      json = send_request(api: 'agent/register', json: MultiJson.encode(register_hash))
+      @token = json[:token] if json && json[:token]
+      sleep 3
+      loop do
+        json = send_request(api: 'agent/poll', params: { token: @token })
+        if json
+          # Promoted or demoted?
+          if json[:token]
+            @token = json[:token] # promote
+          else
+            @token = nil # demote
+          end
+
+          if json[:job]
+            if pid = fork
+              # Parent
+              srand # Ruby 1.8.7 needs reseeding - http://bugs.ruby-lang.org/issues/4338
+              Process.detach(pid)
+            else
+              # Child
+              Dumper::Job.new(self, json[:job]).run_and_exit
+            end
+          end
+        end
+        sleep @token ? 60.seconds : 1.hour
+      end
+    end
+
+    def register_hash
+      {
+        # :pid => Process.pid,
+        # :host => Socket.gethostname,
+        :agent_version => Dumper::VERSION,
+        :app_name => @app_name,
+        :stack => @stack.to_hash,
+      }
+    end
+
+    def send_request(options)
+      uri = URI.parse("#{@api_base}/api/#{options[:api]}")
+      http = Net::HTTP.new(uri.host, uri.port)
+      if uri.is_a? URI::HTTPS
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request['x-app-key'] = @app_key
+      request['x-app-env'] = @app_env
+      request['x-api-version'] = API_VERSION.to_s
+      request['user-agent'] = "Dumper-RailsAgent/#{Dumper::VERSION} (ruby #{::RUBY_VERSION} #{::RUBY_PLATFORM} / rails #{Rails::VERSION::STRING})"
+      if options[:params]
+        request.set_form_data(options[:params])
+      else
+        # Without empty string, WEBrick would complain WEBrick::HTTPStatus::LengthRequired for empty POSTs
+        request.body = options[:json] || ''
+        request['Content-Type'] = 'application/octet-stream'
+      end
+
+      response = http.request(request)
+      if response.code == '200'
+        log response.body, :debug
+      else
+        log '******** ERROR!! ********', :error
+      end
+      MultiJson.decode(response.body).with_indifferent_access
+    rescue
+      log_last_error
+      {} # return empty hash
+    end
+  end
+end

data/lib/dumper/cli.rb

@@ -1,3 +1,6 @@
+Dumper::Dependency.load('thor')
+Dumper::Dependency.load('rainbow')
+
 module Dumper
   class Cli < Thor
     include Thor::Actions

data/lib/dumper/database/base.rb

@@ -0,0 +1,9 @@
+module Dumper
+  module Database
+    class Base
+      def initialize(stack)
+        @stack = stack
+      end
+    end
+  end
+end

data/lib/dumper/database/mysql.rb

@@ -0,0 +1,22 @@
+module Dumper
+  module Database
+    class MySQL < Base
+      def command
+        "mysqldump #{connection_options} #{additional_options} #{@stack.activerecord_config['database']} | gzip"
+      end
+
+      protected
+
+      def connection_options
+        [ :host, :port, :username, :password ].map do |option|
+          next if @stack.activerecord_config[option.to_s].blank?
+          "--#{option}='#{@stack.activerecord_config[option.to_s]}'".gsub('--username', '--user')
+        end.compact.join(' ')
+      end
+
+      def additional_options
+        '--single-transaction'
+      end
+    end
+  end
+end

data/lib/dumper/dependency.rb

@@ -0,0 +1,24 @@
+module Dumper
+  class Dependency
+    LIBS = {
+      'thor' =>       { :require => 'thor',       :version => '~> 0.14.0' },
+      'rainbow' =>    { :require => 'rainbow',    :version => '~> 1.1.4' },
+    }
+
+    def self.load(name)
+      begin
+        gem(name, LIBS[name][:version])
+        require(LIBS[name][:require])
+      rescue LoadError
+        abort <<-EOS
+Dependency missing: #{name}
+To install the gem, issue the following command:
+
+    gem install #{name} -v '#{LIBS[name][:version]}'
+
+Please try again after installing the missing dependency.
+        EOS
+      end
+    end
+  end
+end

data/lib/dumper/job.rb

@@ -0,0 +1,103 @@
+require 'posix/spawn'
+
+module Dumper
+  class Job
+    include POSIX::Spawn
+    include Dumper::Utility::LoggingMethods
+
+    def initialize(agent, job)
+      @agent = agent
+      @stack = agent.stack
+      @job = job
+    end
+
+    def run_and_exit
+      @job[:servers].each do |server|
+        perform(server)
+      end
+    ensure
+      log_last_error if $!
+      log 'exiting...'
+      exit
+    end
+
+    def perform(server)
+      # Prepare
+      json = @agent.send_request(api: 'backup/prepare', params: { server_id: server[:id], manual: server[:manual].to_s })
+      return unless json[:status] == 'ok'
+
+      case server[:type]
+      when 'mysql'
+        @database = Dumper::Database::MySQL.new(@stack)
+      else
+        abort 'invalid server type!' # TBD
+      end
+      backup_id = json[:backup][:id]
+      filename = json[:backup][:filename]
+
+      # Dump
+      start_at = Time.now
+      tempfile = ruby19? ? Tempfile.new(filename, encoding: 'ascii-8bit') : Tempfile.new(filename)
+      log 'starting backup...'
+      log "tempfile = #{tempfile.path}"
+      log "command = #{@database.command}"
+
+      begin
+        pid, stdin, stdout, stderr = popen4(@database.command)
+        stdin.close
+        # Reuse buffer: http://www.ruby-forum.com/topic/134164
+        buffer_size = 1.megabytes
+        buffer = "\x00" * buffer_size # fixed-size malloc optimization
+        while stdout.read(buffer_size, buffer)
+          tempfile.write buffer
+          if tempfile.size > Backup::MAX_FILESIZE
+            raise 'Max filesize exceeded.'
+          end
+        end
+      rescue
+        Process.kill(:INT, pid) rescue SystemCallError
+        @agent.send_request(api: 'backup/fail', params: { backup_id: backup_id, code: 'dump_error', message: $!.to_s })
+        abort
+      ensure
+        [stdin, stdout, stderr].each{|io| io.close unless io.closed? }
+        Process.waitpid(pid)
+      end
+
+      tempfile.flush
+
+      dump_duration = Time.now - start_at
+      log "dump_duration = #{dump_duration}"
+
+      upload_to_s3(json[:url], json[:fields], tempfile.path, filename)
+
+      json = @agent.send_request(api: 'backup/commit', params: { backup_id: backup_id, dump_duration: dump_duration.to_i })
+    rescue
+      log_last_error
+    ensure
+      tempfile.close(true)
+    end
+
+    # Upload
+    def upload_to_s3(url, fields, local_file, remote_file)
+      require 'net/http/post/multipart'
+      fields['file'] = UploadIO.new(local_file, 'application/octet-stream', remote_file)
+      uri = URI.parse(url)
+      request = Net::HTTP::Post::Multipart.new uri.path, fields
+      http = Net::HTTP.new(uri.host, uri.port)
+      if uri.is_a? URI::HTTPS
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      response = http.request(request)
+      log "response from S3 = #{response.to_s}"
+      response
+    rescue
+      log_last_error
+    end
+
+    # Helper
+    def ruby19?
+      RUBY_VERSION >= '1.9.0'
+    end
+  end
+end

data/lib/dumper/stack.rb

@@ -0,0 +1,69 @@
+module Dumper
+  class Stack
+    attr_accessor :rails_env, :dispatcher, :framework, :rackup, :activerecord_config
+
+    def initialize
+      # Rackup?
+      @rackup = find_instance_in_object_space(Rack::Server)
+
+      # Rails?
+      if defined?(::Rails)
+        @framework = :rails
+        @rails_env = Rails.env.to_s
+        @rails_version = Rails::VERSION::STRING
+        @is_supported_rails_version = (::Rails::VERSION::MAJOR >= 3)
+        @activerecord_config = ActiveRecord::Base.configurations[@rails_env]
+      else
+        @framework = :ruby
+      end
+
+      # Which dispatcher?
+      [ :unicorn, :passenger, :thin, :mongrel, :webrick ].find do |name|
+        @dispatcher = send("#{name}?") ? name : nil
+      end
+    end
+
+    def to_hash
+      {
+        framework: @framework,
+        rails_env: @rails_env,
+        rails_version: @rails_version,
+        dispatcher: @dispatcher,
+        activerecord_config: @activerecord_config.reject{|k,v| k == 'password' },
+      }
+    end
+
+    # Compatibility
+    def supported?
+      @is_supported_rails_version && @dispatcher && %w(mysql mysql2).include?(@activerecord_config['adapter'])
+    end
+
+    # Dispatcher
+    def unicorn?
+      defined?(::Unicorn::HttpServer) && find_instance_in_object_space(::Unicorn::HttpServer)
+    end
+
+    def passenger?
+      defined?(::Passenger::AbstractServer) || defined?(::IN_PHUSION_PASSENGER)
+    end
+
+    def thin?
+      # defined?(::Thin::Server) && find_instance_in_object_space(Thin::Server)
+      @rackup and @rackup.server.to_s.demodulize == 'Thin'
+    end
+
+    def mongrel?
+      # defined?(::Mongrel::HttpServer)
+      @rackup and @rackup.server.to_s.demodulize == 'Mongrel'
+    end
+
+    def webrick?
+      # defined?(::WEBrick::VERSION)
+      @rackup and @rackup.server.to_s.demodulize == 'WEBrick'
+    end
+
+    def find_instance_in_object_space(klass)
+      ObjectSpace.each_object(klass).first
+    end
+  end
+end

data/lib/dumper/utility.rb

@@ -1,4 +1,5 @@
 require 'ipaddr'
+require 'logger'
 
 module Dumper
   module Utility
@@ -35,5 +36,37 @@ module Dumper
           IPAddr.new("192.168.0.0/16") ].any?{|i| i.include? self }
       end
     end
+
+    class SlimLogger < ::Logger
+      def initialize(logdev, shift_age = 0, shift_size = 1048576)
+        super
+        self.formatter = SlimFormatter.new
+        self.formatter.datetime_format = "%Y-%m-%dT%H:%M:%S"
+        self.level = Logger::INFO
+      end
+      
+      class SlimFormatter < ::Logger::Formatter
+        # This method is invoked when a log event occurs
+        def call(severity, time, progname, msg)
+          "[%s] %5s : %s\n" % [format_datetime(time), severity, msg2str(msg)]
+        end
+      end
+    end
+
+    module LoggingMethods
+      def logger
+        @logger ||= Dumper::Utility::SlimLogger.new("#{Rails.root}/log/dumper_agent.log", 1, 10.megabytes)
+      end
+
+      def log(msg, level=:info)
+        STDOUT.puts "** [Dumper] " + msg
+        return unless true #should_log?
+        logger.send level, msg
+      end
+
+      def log_last_error
+        log [ $!.class.name, $!.to_s ].join(', ')
+      end
+    end
   end
 end

data/lib/dumper/version.rb

@@ -1,3 +1,3 @@
 module Dumper
-  VERSION = "0.0.3"
+  VERSION = '0.0.4'
 end

data/spec/dumper_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+describe Dumper do
+  it 'initializes' do
+    Dumper::Agent.respond_to?(:start).should be_true
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'dumper'
+
+RSpec.configure do |config|
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dumper
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,17 +9,49 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-29 00:00:00.000000000 Z
+date: 2012-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: posix-spawn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.6
+- !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -28,14 +60,14 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rainbow
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -43,7 +75,8 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Utility that checks the status of a database
+description: Dumper is a backup management system that offers a whole new way to take
+  daily backups of your databases. (coming soon!)
 email:
 - kenn.ejima@gmail.com
 executables:
@@ -52,6 +85,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
 - Gemfile
 - LICENSE
 - README.md
@@ -59,10 +93,18 @@ files:
 - bin/dumper
 - dumper.gemspec
 - lib/dumper.rb
+- lib/dumper/agent.rb
 - lib/dumper/cli.rb
+- lib/dumper/database/base.rb
+- lib/dumper/database/mysql.rb
+- lib/dumper/dependency.rb
+- lib/dumper/job.rb
+- lib/dumper/stack.rb
 - lib/dumper/utility.rb
 - lib/dumper/version.rb
-homepage: ''
+- spec/dumper_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/kenn/dumper
 licenses: []
 post_install_message: 
 rdoc_options: []
@@ -85,6 +127,8 @@ rubyforge_project:
 rubygems_version: 1.8.19
 signing_key: 
 specification_version: 3
-summary: Utility that checks the status of a database
-test_files: []
+summary: The Dumper Agent for Rails (coming soon!)
+test_files:
+- spec/dumper_spec.rb
+- spec/spec_helper.rb
 has_rdoc: