mongo-oplog-backup 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee599dbf9fcfddbf310a1f8196d04e4a9aea58f3
-  data.tar.gz: 60ee3f338862e1a4eecb4aa3c272acb110efc757
+  metadata.gz: 4c98c007c62b80bc6c87b1f141deba9d27933865
+  data.tar.gz: 9a731f45965d4218bbef98e711908d236d41d29c
 SHA512:
-  metadata.gz: 5ddbe8b87d6ef34fdb5b5d0bda352fd41c4a5b6f11a5588d00c9c53e7325d433d40d3dfd3325ae1f03a1f8290039357a8eebbe0a09572131dc04f0daba7c7f92
-  data.tar.gz: 0ad3c9e703de40f772f137c8c7525fb6e049b78d170e3d8ddee2900f7e31f1422e765e4ae8bb94f6e13922fe586fe0f987b4a26ed2820849c654b8e0e480f28e
+  metadata.gz: 5642b1a2790b9d2493c1d5e7f90e8baae53bc6d9c6a81084475d03100903b6cb704124f7a4336060f7c9256900db00a677b06c548c665f5ccdcbae7989f28bea
+  data.tar.gz: cbba467fb5f8c9ffe9c30a79298c6d3779d5e590ebab20bf874139b76b1adb288560caf7910f734093f0dfe5ce5e563a041c8b7ce5bc4af0f75ae8f8d3a3aef6

data/bin/mongo-oplog-backup

@@ -18,6 +18,9 @@ opts = Slop.parse(help: true, strict: true) do
     on :ssl, "Connect to a mongod instance over an SSL connection"
     on :sslAllowInvalidCertificates, "Allow connections to a mongod instance with an invalid certificate"
     on :sslCAFile, "Specifies a Certificate Authority file for validating the SSL certificate provided by the mongod instance.", argument: :required
+    on :sslPEMKeyFile, "Specifies a SSL client certificate and private key for authenticating with the mongod instance.", argument: :required
+    on :sslPEMKeyPassword, "Specifies the password used to open the sslPEMKeyFile, if required.", argument: :required
+    on :authenticationDatabase, "Specifies the database to authenticate against.  This can be used to override the default selection.", argument: :required
     on :host, "Specifies a resolvable hostname for the mongod that you wish to backup.", default: 'localhost', argument: :required
     on :port, "Specifies the port that mongod is running on", default: '27017', argument: :required
     on :u, :username, "Specifies a username to authenticate to the MongoDB instance, if your database requires authentication. Use in conjunction with the --password option to supply a password.", argument: :required
@@ -37,6 +40,9 @@ opts = Slop.parse(help: true, strict: true) do
       config_opts[:username] = opts[:username]
       config_opts[:password] = opts[:password]
       config_opts[:sslCAFile] = opts[:sslCAFile]
+      config_opts[:sslPEMKeyFile] = opts[:sslPEMKeyFile]
+      config_opts[:sslPEMKeyPassword] = opts[:sslPEMKeyPassword]
+      config_opts[:authenticationDatabase] = opts[:authenticationDatabase]
 
       mode = :auto
       if opts.full?
@@ -66,4 +72,60 @@ opts = Slop.parse(help: true, strict: true) do
       puts "mongorestore [--drop] [--gzip] --oplogReplay #{File.join(dir, 'dump')}"
     end
   end
+
+  command 'restore' do
+  banner 'Usage: mongo-oplog-backup restore [options]'
+    on :v, :verbose, 'Enable verbose mode'
+    on :d, :dir, "Directory containing the backup to restore. Must contain a 'dump' folder.", argument: :required
+    on :full, 'Do a full restore, followed by an oplog restore.'
+    on :oplog, 'Restore only the oplogs. Assumes a manual restore has already been done.'
+
+    on :gzip, "Use gzip compression. This option is ignored by Oplog mode, because it autodetects."
+    on :ssl, "Connect to a mongod instance over an SSL connection"
+    on :sslAllowInvalidCertificates, "Allow connections to a mongod instance with an invalid certificate"
+    on :sslCAFile, "Specifies a Certificate Authority file for validating the SSL certificate provided by the mongod instance.", argument: :required
+    on :host, "Specifies a resolvable hostname for the mongod that you wish to backup.", default: 'localhost', argument: :required
+    on :port, "Specifies the port that mongod is running on", default: '27017', argument: :required
+    on :u, :username, "Specifies a username to authenticate to the MongoDB instance, if your database requires authentication. Use in conjunction with the --password option to supply a password.", argument: :required
+    on :p, :password, "Specifies a password to authenticate to the MongoDB instance. Use in conjunction with the --username option to supply a username. Note. the password will not be prompted for, so must be passed as an argument", argument: :required
+
+    on :noIndexRestore, "Don't restore indexes."
+    on :oplogLimit, "Only include oplog entries before the provided Timestamp: <seconds>[:ordinal]", argument: :required
+    on :oplogStartAt, "Ignore oplog batches before the provided Timestamp: <seconds>[:ordinal]. This is useful for resuming a restore after an error.", argument: :required
+
+    run do |opts, args|
+      dir = opts[:dir]
+      raise ArgumentError, 'dir must be specified' unless dir
+      raise ArgumentError, 'dir must contain a dump subfolder' unless File.directory?(File.join(dir, 'dump'))
+      config_opts = {
+        dir: dir,
+        ssl: opts.ssl?,
+        gzip: opts.gzip?,
+        sslAllowInvalidCertificates: opts.sslAllowInvalidCertificates?,
+        noIndexRestore: opts.noIndexRestore?
+      }
+      config_opts[:host] = opts[:host]
+      config_opts[:port] = opts[:port]
+      config_opts[:username] = opts[:username]
+      config_opts[:password] = opts[:password]
+      config_opts[:sslCAFile] = opts[:sslCAFile]
+      config_opts[:oplogLimit] = opts[:oplogLimit]
+      config_opts[:oplogStartAt] = opts[:oplogStartAt]
+
+      if opts.full?
+        mode = :full
+      elsif opts.oplog?
+        mode = :oplog
+      else
+        raise ArgumentError, "either full or oplog must be specified"
+      end
+
+      restore_opts = {}
+      restore_opts[:oplogLimit] = opts[:oplogLimit]
+
+      config = MongoOplogBackup::Config.new(config_opts)
+      restore = MongoOplogBackup::Restore.new(config)
+      restore.perform(mode, restore_opts)
+    end
+  end
 end

data/lib/mongo_oplog_backup.rb

@@ -8,6 +8,7 @@ require 'mongo_oplog_backup/command'
 require 'mongo_oplog_backup/config'
 require 'mongo_oplog_backup/backup'
 require 'mongo_oplog_backup/oplog'
+require 'mongo_oplog_backup/restore'
 
 module MongoOplogBackup
   def self.log

data/lib/mongo_oplog_backup/config.rb

@@ -18,6 +18,9 @@ module MongoOplogBackup
         options[:ssl] = conf["ssl"] unless conf["ssl"].nil?
         options[:sslAllowInvalidCertificates] = conf["sslAllowInvalidCertificates"] unless conf["sslAllowInvalidCertificates"].nil?
         options[:sslCAFile] = conf["sslCAFile"] unless conf["sslCAFile"].nil?
+        options[:sslPEMKeyFile] = conf["sslPEMKeyFile"] unless conf["sslPEMKeyFile"].nil?
+        options[:sslPEMKeyPassword] = conf["sslPEMKeyPassword"] unless conf["sslPEMKeyPassword"].nil?
+        options[:authenticationDatabase] = conf["authenticationDatabase"] unless conf["authenticationDatabase"].nil?
         options[:host] = conf["host"] unless conf["host"].nil?
         options[:port] = conf["port"].to_s unless conf["port"].nil?
         options[:username] = conf["username"] unless conf["username"].nil?
@@ -39,11 +42,18 @@ module MongoOplogBackup
       args = []
       args << '--ssl' if options[:ssl]
       args << '--sslAllowInvalidCertificates' if options[:sslAllowInvalidCertificates]
-      [:host, :port, :username, :password, :sslCAFile].each do |option|
+      [:host, :port, :username, :password, :sslCAFile, :sslPEMKeyFile, :sslPEMKeyPassword].each do |option|
         args += ["--#{option}", options[option].strip] if options[option]
       end
+      
+      if options[:authenticationDatabase]
+        args += ['--authenticationDatabase', options[:authenticationDatabase]]
+      else
+        args += ['--authenticationDatabase', 'admin'] if options[:username] && !options[:sslPEMKeyFile]
+        args += ['--authenticationDatabase', '$external'] if options[:sslPEMKeyFile]
+      end
 
-      args += ['--authenticationDatabase', 'admin'] if options[:username]
+      args += ['--authenticationMechanism', 'MONGODB-X509'] if options[:sslPEMKeyFile]
 
       args
     end
@@ -78,7 +88,11 @@ module MongoOplogBackup
     end
 
     def mongo(db, script)
-      exec(['mongo'] + command_line_options + ['--quiet', '--norc', script])
+      exec(['mongo'] + command_line_options + ['--quiet', '--norc', db, script])
+    end
+
+    def mongorestore(*args)
+      exec(['mongorestore'] + command_line_options + args.flatten)
     end
 
     def command_string(cmd)

data/lib/mongo_oplog_backup/ext/timestamp.rb

@@ -21,10 +21,20 @@ module MongoOplogBackup::Ext
     end
 
     module ClassMethods
-      # Accepts {'t' => seconds, 'i' => increment}
+      # Accepts {'t' => seconds, 'i' => increment} or {'$timestamp' => {'t' => seconds, 'i' => increment}}
       def from_json(data)
+        data = data['$timestamp'] if data['$timestamp']
         self.new(data['t'], data['i'])
       end
+
+      # Accepts: <seconds>[:ordinal]
+      def from_string(string)
+        match = /(\d+)(?::(\d+))?/.match(string)
+        return nil unless match
+        s1 = match[1].to_i
+        i1 = match[2].to_i
+        self.new(s1,i1)
+      end
     end
 
   end

data/lib/mongo_oplog_backup/restore.rb

@@ -0,0 +1,141 @@
+require 'json'
+require 'fileutils'
+require 'mongo_oplog_backup/oplog'
+
+module MongoOplogBackup
+  class Restore
+    attr_reader :config
+
+    def backup_folder
+      File.join(config.backup_dir)
+    end
+
+    def oplog_restore_folder
+      File.join(backup_folder, 'tmp-restore')
+    end
+
+    def initialize(config)
+      @config = config
+    end
+
+    # Given a directory of oplog dumps generated by the backup feature,
+    # iteratively mongorestore them.
+    # Mongorestore <3.4 expects a file named oplog.bson the directory specified
+    # Mongorestore 3.4 adds support for --oplogFile parameter, which simplifies things.
+    def restore_oplogs(dir, options={})
+      default_restore_args = ['--stopOnError', '--oplogReplay']
+      default_restore_args << '--noIndexRestore' if !!config.options[:noIndexRestore]
+      if options[:oplogLimit]
+        default_restore_args += ['--oplogLimit', options[:oplogLimit]]
+        oplog_limit = BSON::Timestamp.from_string(options[:oplogLimit])
+      end
+
+      oplog_start_at = BSON::Timestamp.from_string(config.options[:oplogStartAt]) if config.options[:oplogStartAt]
+
+      source_files = Oplog.find_oplogs(dir)
+
+      validate_continuity!(source_files)
+
+      MongoOplogBackup.log.debug "Starting oplog restore..."
+      source_files.each do |filename|
+        # TODO: mongorestore 3.4 supports --oplogFile
+
+        unless oplog_start_at.nil?
+          timestamps = Oplog.timestamps_from_filename(filename)
+          if timestamps[:last] <= oplog_start_at
+            MongoOplogBackup.log.debug "Skipping batch: #{filename}. Last op in batch (ts: #{timestamps[:last]}) is before oplogStartAt: #{oplog_start_at}"
+            next
+          end
+        end
+        unless oplog_limit.nil?
+          timestamps = Oplog.timestamps_from_filename(filename)
+          if timestamps[:first] > oplog_limit
+            MongoOplogBackup.log.debug "Skipping batch: #{filename}. First op in batch (ts: #{timestamps[:first]}) is after oplogLimit: #{oplog_limit}"
+            next
+          end
+        end
+
+        temp_file_path = create_temp_oplog_dir(dir, filename)
+        oplog_dir_path = File.dirname(temp_file_path)
+
+        restore_args = default_restore_args.dup
+        restore_args << '--gzip' if Oplog.gzip_fingerprint(filename)
+        restore_args << oplog_dir_path
+        status = config.mongorestore(restore_args).status
+        if status != 0
+          MongoOplogBackup.log.error("Mongorestore failed during oplog restore. Aborting. Exit code: #{status}")
+          raise 'Oplog restore failed.'
+        end
+
+        begin
+          File.unlink(temp_file_path)
+          Dir.rmdir(oplog_dir_path)
+        rescue SystemCallError => e
+          MongoOplogBackup.log.error("Clean-up error for '#{temp_file_path}. #{e.message}")
+          raise
+        end
+      end
+      MongoOplogBackup.log.debug "Oplog restore complete."
+    end
+
+    def restore_dump(dir, options={})
+      restore_args = ['--stopOnError']
+      restore_args << '--noIndexRestore' if !!config.options[:noIndexRestore]
+      restore_args << '--gzip' if config.use_compression?
+      restore_args << File.join(dir, 'dump')
+
+      MongoOplogBackup.log.debug "Starting full restore..."
+      status = config.mongorestore(restore_args).status
+      if status != 0
+        MongoOplogBackup.log.error("Mongorestore failed.")
+        raise 'Full restore failed.'
+      end
+      MongoOplogBackup.log.debug "Full restore complete."
+    end
+
+    def perform(mode, options={})
+      if options[:oplogLimit]
+        raise ArgumentError, "oplogLimit is not a timestamp: eg. <seconds>[:ordinal]" unless options[:oplogLimit] =~ /\A\d+(?::\d+)?\z/
+      end
+
+      if mode == :oplog
+        restore_oplogs(backup_folder, options)
+      elsif mode == :full
+        restore_dump(backup_folder, options)
+        restore_oplogs(backup_folder, options)
+      end
+    end
+
+    private
+
+    # Check for gaps in timestamps based only on the filename, trusting that the oplog set is complete (ie. the filename timestamps match the actual data)
+    def validate_continuity!(source_files)
+      expected_first = nil
+      MongoOplogBackup.log.debug "Checking timestamps..."
+      source_files.each do |filename|
+        timestamps = Oplog.timestamps_from_filename(filename)
+        raise("Filename without timestamps found in restore set: #{filename}") if timestamps.nil?
+        if expected_first && expected_first != timestamps[:first]
+          raise "Missing oplog dump? Expected a filename with first timestamp '#{expected_first}', but got '#{timestamps[:first]}' from filename '#{filename}'."
+        end
+        expected_first = timestamps[:last]
+      end
+      true
+    end
+
+    def create_temp_oplog_dir dir, filename
+      temp_dir = File.join(dir, 'tmp-restore', File.basename(filename))
+      temp_file_path = File.join(temp_dir, 'oplog.bson')
+      FileUtils.mkdir_p(temp_dir)
+      File.link(filename, temp_file_path)
+      return temp_file_path
+    rescue Errno::EEXIST => e
+      # Probably, 'new_name' already exists when creating link.
+      MongoOplogBackup.log.warn("Temporary oplog.bson link already exists: #{e.message}. Assuming valid.")
+      return temp_file_path
+    rescue SystemCallError => e
+      MongoOplogBackup.log.error("Setup error for oplog '#{filename}' in '#{temp_file_path}")
+      raise
+    end
+  end
+end

data/lib/mongo_oplog_backup/version.rb

@@ -1,3 +1,3 @@
 module MongoOplogBackup
-  VERSION = "0.0.11"
+  VERSION = "0.0.12"
 end

data/spec/timestamp_spec.rb

@@ -41,6 +41,15 @@ describe MongoOplogBackup::Ext::Timestamp do
     ts.as_json.should == json
   end
 
+  it 'should handle extended-json timestamp format' do
+    json = { "t" => 1408004593, "i" => 20}
+    extendedjson = {"$timestamp" => json}
+    ts = BSON::Timestamp.from_json(extendedjson)
+    ts.seconds.should == 1408004593
+    ts.increment.should == 20
+    ts.as_json.should == json
+  end
+
   it 'should define to_s' do
     ts = BSON::Timestamp.new(1408004593, 2)
     ts.to_s.should == '1408004593:2'

data/testing.sh

@@ -0,0 +1,57 @@
+#!/bin/bash -ex
+
+echo "This requires a mongodb instance running on 27017."
+echo "It will first SHUTDOWN any instances already running on that port, and then start new ones."
+echo "If this is undesired, CTRL+C within 5 seconds..." 
+sleep 5
+echo "Here we go..."
+mongo --port 27017 admin --eval 'db.shutdownServer({force: true})' || true
+sleep 5
+rm -rf backup-test/*
+rm -rf testdb
+mkdir testdb
+mongod --port 27017 --dbpath testdb --replSet rs0 --oplogSize 20 --noprealloc --fork --smallfiles --logpath mongodb.log
+sleep 3
+mongo --port 27017 admin --eval 'printjson(rs.initiate());'
+sleep 20
+
+
+
+bundle exec rspec
+
+
+
+bundle exec bin/mongo-oplog-backup backup --port 27017 --dir backup-test/ --gzip --full
+mongo --port 27017 backup-test --eval 'db.test.insert({"a":2})'
+
+bundle exec bin/mongo-oplog-backup backup --port 27017 --dir backup-test/ --gzip --oplog
+
+sleep 5
+mongo --port 27017 backup-test --eval 'db.test.insert({"a":3})'
+bundle exec bin/mongo-oplog-backup backup --port 27017 --dir backup-test/ --oplog
+
+
+sleep 5
+mongo --port 27017 backup-test --eval 'db.test.insert({"a":4})'
+bundle exec bin/mongo-oplog-backup backup --port 27017 --dir backup-test/ --oplog
+
+
+
+mongo --port 27017 admin --eval 'db.shutdownServer({force: true})'
+sleep 5
+rm -rf testdb/*
+mongod --port 27017 --dbpath testdb --replSet rs0 --oplogSize 20 --noprealloc --fork --smallfiles --logpath mongodb.log
+sleep 3
+mongo --port 27017 admin --eval 'printjson(rs.initiate());'
+sleep 20
+
+export BACKUPDIR=`ls -1t backup-test/ |grep backup- |head -n 1`
+
+bundle exec bin/mongo-oplog-backup restore --full --gzip --dir backup-test/$BACKUPDIR --port 27017
+mongo --port 27017 backup-test --eval 'db.test.find()'
+
+
+#mongorestore --gzip --port 27017 backup-test/$BACKUPDIR/dump
+#mongo --port 27017 backup-test --eval 'db.test.find()'
+#bundle exec bin/mongo-oplog-backup restore --oplog --dir backup-test/$BACKUPDIR --port 27017
+#mongo --port 27017 backup-test --eval 'db.test.find()'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mongo-oplog-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Ralf Kistner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-25 00:00:00.000000000 Z
+date: 2017-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson
@@ -118,6 +118,7 @@ files:
 - lib/mongo_oplog_backup/ext/enumerable.rb
 - lib/mongo_oplog_backup/ext/timestamp.rb
 - lib/mongo_oplog_backup/oplog.rb
+- lib/mongo_oplog_backup/restore.rb
 - lib/mongo_oplog_backup/version.rb
 - mongo-oplog-backup.gemspec
 - oplog-last-timestamp.js
@@ -136,6 +137,7 @@ files:
 - spec/oplog_spec.rb
 - spec/spec_helper.rb
 - spec/timestamp_spec.rb
+- testing.sh
 homepage: ''
 licenses:
 - MIT