mongo-oplog-backup 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c464ac7e971978efe22788614317405be8935f25
-  data.tar.gz: a12be71eeb235821321b8946ef6bcf10ff040724
+  metadata.gz: 6bad50ae17686f3dadca7ed09faa925433f2ba4f
+  data.tar.gz: eb9e08a62cdddf9b0490d4903f62da08b47e19cf
 SHA512:
-  metadata.gz: 2e495dcd5d7ebf02816187845dd7424980e77b5f098828dee1155d88fe59649d196d342ad5de048adb7631dd4faf98cc798cc2bc0180fde46ae3be5beb78e64c
-  data.tar.gz: accab78d844a1e729e0a1dc99264def3e07c31509344d337a32fe7bdb35c6f95288037bcc1e919004fba67514da4672fbbded878fdfe80885ccd365d35348789
+  metadata.gz: 44a3a86eab76a36be2a0d25eda220be4a12b4d597987f5e8221da365aa11d17d58db0b8e848be588cf161654a9f944cfd1e800c0e1def211b9df779b0195e35a
+  data.tar.gz: a79050e079b5f40a48ea2865044aa075aa2f10c91b53191ff78aa2a604bb7ee95efa41189cda21ee7f06c12c0f458026cdded37906eb131842270cba42445b19

data/lib/mongo_oplog_backup.rb

@@ -4,6 +4,7 @@ require 'mongo_oplog_backup/version'
 require 'mongo_oplog_backup/ext/enumerable'
 require 'mongo_oplog_backup/ext/timestamp'
 
+require 'mongo_oplog_backup/command'
 require 'mongo_oplog_backup/config'
 require 'mongo_oplog_backup/backup'
 require 'mongo_oplog_backup/oplog'
@@ -15,6 +16,7 @@ module MongoOplogBackup
 
   def self.log= log
     @@log = log
+    Command.logger = log
   end
 
   @@log = Logger.new STDOUT

data/lib/mongo_oplog_backup/backup.rb

@@ -30,11 +30,13 @@ module MongoOplogBackup
       raise ArgumentError, ":start is required" unless start_at
 
       if start_at
-        query = "--query \"{ts : { \\$gte : { \\$timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}\""
+        query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]
       else
-        query = ""
+        query = []
       end
-      config.mongodump("--out #{config.oplog_dump_folder} --db local --collection oplog.rs #{query}")
+      config.mongodump(['--out', config.oplog_dump_folder,
+        '--db', 'local', '--collection', 'oplog.rs'] +
+        query)
 
       unless File.exists? config.oplog_dump
         raise "mongodump failed"
@@ -82,7 +84,7 @@ module MongoOplogBackup
 
     def latest_oplog_timestamp
       script = File.expand_path('../../oplog-last-timestamp.js', File.dirname(__FILE__))
-      result_text = config.mongo('admin', script)
+      result_text = config.mongo('admin', script).standard_output
       begin
         response = JSON.parse(result_text)
         return nil unless response['position']
@@ -97,8 +99,10 @@ module MongoOplogBackup
       raise "Cannot backup with empty oplog" if position.nil?
       backup_name = "backup-#{position}"
       dump_folder = File.join(config.backup_dir, backup_name, 'dump')
-      # TODO: fail hard if this command fails.
-      config.mongodump("--out #{dump_folder}")
+      config.mongodump('--out', dump_folder)
+      unless File.directory? dump_folder
+        raise 'Full backup failed'
+      end
       return {
         position: position,
         backup: backup_name

data/lib/mongo_oplog_backup/command.rb

@@ -0,0 +1,124 @@
+require 'open3'
+require 'io/wait'
+
+
+module MongoOplogBackup
+  class Command
+    def self.logger= logger
+      @logger = logger
+    end
+
+    def self.logger
+      @logger
+    end
+
+    attr_reader :command
+    attr_reader :standard_output
+    attr_reader :standard_error
+    attr_reader :status
+
+    def self.execute(command, options={})
+      Command.new(command, options).run
+    end
+
+    # command must be an array containing the command and arguments
+    def initialize(command, options={})
+      @command = command
+      @standard_output = ''
+      @standard_error = ''
+      @out_blocks = []
+      @err_blocks = []
+
+      logger = options[:logger] || Command.logger
+
+      if logger
+        log_output(logger)
+      end
+
+      on_stdout do |data|
+        @standard_output << data
+      end
+      on_stderr do |data|
+        @standard_error << data
+      end
+    end
+
+    def on_stdout_line &block
+      on_stdout(&lines_proc(&block))
+    end
+
+    def on_stderr_line &block
+      on_stderr(&lines_proc(&block))
+    end
+
+    def on_stderr &block
+      @err_blocks << block
+    end
+
+    def on_stdout &block
+      @out_blocks << block
+    end
+
+    def log_output(logger)
+      on_stdout_line do |line|
+        logger.debug(line)
+      end
+      on_stderr_line do |line|
+        logger.error(line)
+      end
+    end
+
+    def run
+      @status = Open3.popen3(*command) do |stdin, stdout, stderr, wait_thr|
+        stdin.close_write
+        until all_eof([stdout, stderr])
+          read_available_data(stdout) do |data|
+            @out_blocks.each do |block|
+              block.call(data)
+            end
+          end
+          read_available_data(stderr) do |data|
+            @err_blocks.each do |block|
+              block.call(data)
+            end
+          end
+          sleep 0.001
+        end
+
+        wait_thr.value
+      end
+      raise!
+      self
+    end
+
+    def raise!
+      unless status.success?
+        raise "Command failed with exit code #{status.exitstatus}"
+      end
+      self
+    end
+
+    private
+    def lines_proc &block
+      # TODO: buffer partial lines
+      return Proc.new do |data|
+        data.split("\n").each do |line|
+          block.call line
+        end
+      end
+    end
+
+    BLOCK_SIZE = 1024
+
+    def read_available_data(io, &block)
+      if io.ready?
+        data = io.read_nonblock(BLOCK_SIZE)
+        block.call data
+      end
+    end
+
+    def all_eof(files)
+      files.find { |f| !f.eof }.nil?
+    end
+  end
+end

data/lib/mongo_oplog_backup/config.rb

@@ -1,3 +1,5 @@
+require 'shellwords'
+
 module MongoOplogBackup
   class Config
     attr_reader :options
@@ -18,7 +20,7 @@ module MongoOplogBackup
         options[:username] = conf["username"] unless conf["username"].nil?
         options[:password] = conf["password"] unless conf["password"].nil?
       end
-      
+
       options
     end
 
@@ -27,14 +29,15 @@ module MongoOplogBackup
     end
 
     def command_line_options
-      ssl = options[:ssl] ? '--ssl ' : ''
-      host = options[:host] ? "--host #{options[:host].strip} " : ''
-      port = options[:port] ? "--port #{options[:port].strip} " : ''
-      username = options[:username] ? "--username #{options[:username].strip} " : ''
-      password = options[:password] ? "--password #{options[:password].strip} " : ''
-      # TODO: make this configurable?
-      authdb = options[:username] ? '--authenticationDatabase admin ' : ''
-      "#{host}#{port}#{ssl}#{username}#{password}"
+      args = []
+      args << '--ssl' if options[:ssl]
+      [:host, :port, :username, :password].each do |option|
+        args += ["--#{option}", options[option].strip] if options[option]
+      end
+
+      args += ['--authenticationDatabase', 'admin'] if options[:username]
+
+      args
     end
 
     def oplog_dump_folder
@@ -54,16 +57,30 @@ module MongoOplogBackup
     end
 
     def exec(cmd)
-      MongoOplogBackup.log.debug ">>> #{cmd}"
-      `#{cmd}`
+      MongoOplogBackup.log.debug ">>> #{command_string(cmd)}"
+      Command.execute(cmd)
     end
 
-    def mongodump(args)
-      MongoOplogBackup.log.info exec("mongodump #{command_line_options} #{args}")
+    def mongodump(*args)
+      exec(['mongodump'] + command_line_options + args.flatten)
     end
 
     def mongo(db, script)
-      exec("mongo #{command_line_options} --quiet --norc #{db} #{script}")
+      exec(['mongo'] + command_line_options + ['--quiet', '--norc', script])
+    end
+
+    def command_string(cmd)
+      previous = nil
+      filtered = cmd.map do |token|
+        pwd = (previous == '--password')
+        previous = token
+        if pwd
+          '***'
+        else
+          token
+        end
+      end
+      filtered.shelljoin
     end
   end
 end

data/lib/mongo_oplog_backup/version.rb

@@ -1,3 +1,3 @@
 module MongoOplogBackup
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/spec/command_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe MongoOplogBackup::Command do
+  it 'should get stdout' do
+    result = MongoOplogBackup::Command.execute(['echo', 'something'])
+    result.standard_output.should == "something\n"
+    result.standard_error.should == ""
+    result.status.exitstatus.should == 0
+  end
+
+  it 'should get stderr' do
+    result = MongoOplogBackup::Command.execute(['ruby', '-e', '$stderr.puts "FOO"'])
+    result.standard_output.should == ""
+    result.standard_error.should == "FOO\n"
+    result.status.exitstatus.should == 0
+  end
+
+  it 'should raise on a non-zero exit code' do
+    command = MongoOplogBackup::Command.new(['ruby', '-e', 'exit 123'])
+    -> { command.run }.should raise_error
+    command.status.exitstatus.should == 123
+  end
+
+  it 'should log' do
+    io = StringIO.new
+    logger = Logger.new io
+    MongoOplogBackup::Command.execute(['ruby', '-e', 'puts "BAR"; $stderr.puts "FOO"'], logger: logger)
+    io.rewind
+    log = io.read
+    log.should =~ /D, \[.+\] DEBUG -- : BAR\nE, \[.+\] ERROR -- : FOO\n/
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mongo-oplog-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Ralf Kistner
@@ -113,6 +113,7 @@ files:
 - bin/mongo-oplog-backup
 - lib/mongo_oplog_backup.rb
 - lib/mongo_oplog_backup/backup.rb
+- lib/mongo_oplog_backup/command.rb
 - lib/mongo_oplog_backup/config.rb
 - lib/mongo_oplog_backup/ext/enumerable.rb
 - lib/mongo_oplog_backup/ext/timestamp.rb
@@ -122,6 +123,7 @@ files:
 - oplog-last-timestamp.js
 - sample-config.yml
 - spec/backup_spec.rb
+- spec/command_spec.rb
 - spec/enumerable_spec.rb
 - spec/fixtures/oplog-1408088734:1-1408088740:1.bson
 - spec/fixtures/oplog-1408088740:1-1408088810:1.bson
@@ -156,6 +158,7 @@ specification_version: 4
 summary: Incremental backups for MongoDB using the oplog.
 test_files:
 - spec/backup_spec.rb
+- spec/command_spec.rb
 - spec/enumerable_spec.rb
 - spec/fixtures/oplog-1408088734:1-1408088740:1.bson
 - spec/fixtures/oplog-1408088740:1-1408088810:1.bson