dumper 1.3.7 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6b408bf82e51cfbc886c06ef6f6bb3466fd49a7
-  data.tar.gz: c5156f36b7f7d5a29cee37008db59a74ba307c94
+  metadata.gz: 821b7a382e2fe27d7fe8fb1c3d4368368d3bea54
+  data.tar.gz: fccb695b3b21ac6998bf8423ca891be8033398db
 SHA512:
-  metadata.gz: fda0aa66cdeaad61af0e83d6b9ca21f3df63ffa7371d0c440df842d11dadb22b9f24548bded0f62595936c0db115a7388ebf57e8510488d00846145ad407b6c2
-  data.tar.gz: 7a3852e30755078dec633d9da7b6a494b4fe10c77816555119e18ebde86fe9ed93b1ca644b0f741fe104f4b76dece96f3891c88291192e698c695428d8a9bf24
+  metadata.gz: 31f45ae50d8441350d87e90168cec56054c1295851f6066d0ca852cf9263938fa2c33eeb02223554fa2889dd126836fb24d0306f9b63d39f71e79e4f60332500
+  data.tar.gz: 424fedc83af11fa9a493a1ab12a8a5391068f34ff4a14aa2d251003e800d33969cdc0ad0c00d67d7574682c1154b70e5777464b43e410a6daffe8466c5450632

data/lib/dumper/cli.rb

@@ -1,5 +1,6 @@
 Dumper::Dependency.load('thor')
 Dumper::Dependency.load('rainbow')
+Dumper::Dependency.load('net-ntp')
 
 module Dumper
   class Cli < Thor
@@ -9,23 +10,23 @@ module Dumper
     def doctor
       check_ip
       check_cnf
+      check_clock
     end
 
     no_tasks do
       def check_ip
-        puts 'Checking IP address...'
+        print 'Checking IP address... '
         @ip = Dumper::Utility::IP.new
-        str = "#{@ip.ip} ... "
+        print "#{@ip.ip} => "
         if @ip.ipaddr.private?
-          str << 'private'.color(:red)
+          puts "private IP, #{fetch_will_fail_warning}".color(:red)
         else
-          str << 'public'.color(:green)
+          puts 'public IP, good'.color(:green)
         end
-        puts str
       end
 
       def check_cnf
-        puts 'Checking my.cnf...'
+        print 'Checking my.cnf... '
         bound = nil
         ['/etc/my.cnf', '/etc/mysql/my.cnf', '/usr/etc/my.cnf', '~/.my.cnf'].each do |name|
           fullpath = File.expand_path(name)
@@ -39,14 +40,32 @@ module Dumper
         end
         if bound
           if bound == '127.0.0.1'
-            puts 'There is bind-address = 127.0.0.1 ... ' << 'fail'.color(:red)
+            print 'There is bind-address = 127.0.0.1 => '
           elsif IPAddr.new(bound).private?
-            puts "There is bind-address = #{bound} ... " << 'fail'.color(:red)
+            print "There is bind-address = #{bound} => "
           end
+          puts fetch_will_fail_warning.color(:red)
         else
-          puts 'No bind-address defined in my.cnf ... ' << 'ok'.color(:green)
+          puts 'No bind-address defined in my.cnf => ' << 'good'.color(:green)
         end
       end
+
+      def check_clock
+        print 'Checking server clock accuracy... '
+        target = Net::NTP.get('us.pool.ntp.org').time
+        source = Time.now
+        diff = (target - source).abs.round(3)
+        print "#{source.strftime('%Y-%m-%d %H:%M:%S')} (server time) vs #{target.strftime('%Y-%m-%d %H:%M:%S')} (ntp time), diff: #{diff} seconds => "
+        if diff > 15 * 60
+          puts 'warning, Amazon S3 does not accept clock skewed more than 15 minutes.'.color(:red)
+        else
+          puts 'good'.color(:green)
+        end
+      end
+
+      def fetch_will_fail_warning
+        'warning - fetch from dumper.io to this server will fail, you will need to use the dumper gem with rails.'
+      end
     end
   end
 end

data/lib/dumper/dependency.rb

@@ -1,8 +1,9 @@
 module Dumper
   class Dependency
     LIBS = {
-      'thor' =>       { :require => 'thor',       :version => '~> 0.14.0' },
+      'thor' =>       { :require => 'thor',       :version => '~> 0.14' },
       'rainbow' =>    { :require => 'rainbow',    :version => '~> 1.1.4' },
+      'net-ntp' =>    { :require => 'net/ntp',    :version => '~> 2.1.1' },
     }
 
     def self.load(name)

data/lib/dumper/job.rb

@@ -55,12 +55,13 @@ module Dumper
       end
 
       dump_duration = Time.now - start_at
-      log "dump_duration = #{dump_duration}"
-      if (filesize = File.size(@database.dump_path)) > @agent.max_filesize
-        abort_with("max filesize exceeded: #{filesize}", :too_large)
+      @filesize = File.size(@database.dump_path)
+      log "dump_duration = #{dump_duration}, filesize = #{@filesize}"
+      if @filesize > @agent.max_filesize
+        abort_with("max filesize exceeded: #{@filesize}", :too_large)
       end
 
-      upload_to_s3(json[:url], json[:fields])
+      upload_to_s3(json)
 
       json = @agent.api_request('backup/commit', :params => { :backup_id => @backup_id, :dump_duration => dump_duration.to_i })
     rescue
@@ -70,10 +71,37 @@ module Dumper
     end
 
     # Upload
-    def upload_to_s3(url, fields)
+    def upload_to_s3(json)
+      if defined?(AWS::S3::MultipartUpload) and @filesize > 100.megabytes
+        # aws-sdk gem installed
+        upload_by_aws_sdk(json)
+      else
+        # fallback to multipart-post
+        upload_by_multipart_post(json)
+      end
+    rescue
+      abort_with("upload error: #{$!} - please contact our support.", :upload_error)
+    end
+
+    def upload_by_aws_sdk(json)
+      log "uploading by aws-sdk v#{AWS::VERSION}"
+
+      s3 = json[:s3_federation]
+      aws = AWS::S3.new(s3[:credentials])
+      aws.buckets[s3[:bucket]].objects[s3[:key]].write(
+        file: @database.dump_path,
+        content_type: 'application/octet-stream',
+        content_disposition: "attachment; filename=#{@database.filename}",
+      )
+    end
+
+    def upload_by_multipart_post(json)
       require 'net/http/post/multipart'
+      log "uploading by multipart-post v#{Gem.loaded_specs['multipart-post'].version.to_s}"
+
+      fields = json[:fields]
       fields['file'] = UploadIO.new(@database.dump_path, 'application/octet-stream', @database.filename)
-      uri = URI.parse(url)
+      uri = URI.parse(json[:url])
       request = Net::HTTP::Post::Multipart.new uri.path, fields
       http = Net::HTTP.new(uri.host, uri.port)
       if uri.is_a? URI::HTTPS
@@ -102,8 +130,6 @@ module Dumper
       else
         abort_with("upload error: #{response.to_s} - #{response.body}", :upload_error)
       end
-    rescue
-      abort_with("upload error: #{$!}", :upload_error)
     end
 
     def abort_with(text, code=nil)

data/lib/dumper/version.rb

@@ -1,3 +1,3 @@
 module Dumper
-  VERSION = '1.3.7'
+  VERSION = '1.4.0'
 end

data/spec/dumper_spec.rb

@@ -7,6 +7,23 @@ describe Dumper do
     Dumper::Agent.respond_to?(:start).should be_true
   end
 
+  it 'loads everything' do
+    expect {
+      Dumper::Agent
+      # Dumper::Cli
+      Dumper::Dependency
+      Dumper::Job
+      Dumper::Stack
+      Dumper::Utility
+      Dumper::VERSION
+      Dumper::Database::Base
+      Dumper::Database::MySQL
+      Dumper::Database::PostgreSQL
+      Dumper::Database::MongoDB
+      Dumper::Database::Redis
+    }.to_not raise_error
+  end
+
   describe :Stack do
     it 'initializes stack' do
       stack = Dumper::Stack.new

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dumper
 version: !ruby/object:Gem::Version
-  version: 1.3.7
+  version: 1.4.0
 platform: ruby
 authors:
 - Kenn Ejima
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-13 00:00:00.000000000 Z
+date: 2013-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json