mongo-oplog-backup 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ee5f2e07aafef152cc9122c0857a8546159fb1e
-  data.tar.gz: 29c70641a560009a829822a0925c827ca9b24854
+  metadata.gz: 0c7c7a2d6fe026794174767a87b18f6ceb6e3a23
+  data.tar.gz: f08e6267096e438fc98c2f35e09984dfede204f9
 SHA512:
-  metadata.gz: da1308e0f5e15a7a3b89816cdea169bf60801f2adecb5d4ae8a14fcdb46a858aa1fb3ec455e9af73ba332af1f55932416859b9ba638dba5d51fe8bb3274727d4
-  data.tar.gz: 4728871d1ca7b05fec204ddb66c5a66918fc4a3e4c9cddc89c415764f03d074d4426cd0704a5c94f8abef244894bff5aede96f0c9f2e8d7fee9cd7e624a5c5a1
+  metadata.gz: 584ccd474c2601256d71c831623e903d80505f549787b78d4cbfa71375af35d630616ff5bad7e0b20af4729dbf2e204cbcef280cf73e72d28e62fe019c995d28
+  data.tar.gz: 4db03ccb06d66a6295fde3c79a75066a4ded1eb3830cd0416e46b9f7a42c99897a88a0d71035100e61960d1629c0e8291447be0f4e42157413bcdaf1da7bb78a

data/README.md

@@ -14,7 +14,7 @@ a single file that can be stored on your preferred medium, for example Amazon S3
 or an FTP site. This project only provides the tools to produce the backup files,
 and it's up to you to transfer it to a backup medium.
 
-Interally the `mongodump` command is used for the backup operations. Initially
+Internally the `mongodump` command is used for the backup operations. Initially
 a full dump is performed, after which incremetal backups are performed by backing
 up new sections of the oplog. Only the standard BSON format from mongodump is used.
 

data/bin/mongo-oplog-backup

@@ -14,6 +14,7 @@ opts = Slop.parse(help: true, strict: true) do
     on :oplog, 'Force oplog backup'
 
     on :f, :file, 'Configuration file for common defaults', argument: :required
+    on :gzip, "Use gzip compression"
     on :ssl, "Connect to a mongod instance over an SSL connection"
     on :sslAllowInvalidCertificates, "Allow connections to a mongod instance with an invalid certificate"
     on :sslCAFile, "Specifies a Certificate Authority file for validating the SSL certificate provided by the mongod instance.", argument: :required
@@ -26,6 +27,7 @@ opts = Slop.parse(help: true, strict: true) do
       dir = opts[:dir] || 'backup'
       config_opts = {
         dir: dir,
+        gzip: opts.gzip?,
         ssl: opts.ssl?,
         sslAllowInvalidCertificates: opts.sslAllowInvalidCertificates?
       }
@@ -61,7 +63,7 @@ opts = Slop.parse(help: true, strict: true) do
       MongoOplogBackup::Oplog.merge_backup(dir)
       puts
       puts "Restore the backup with: "
-      puts "mongorestore [--drop] --oplogReplay #{File.join(dir, 'dump')}"
+      puts "mongorestore [--drop] [--gzip] --oplogReplay #{File.join(dir, 'dump')}"
     end
   end
 end

data/lib/mongo_oplog_backup/backup.rb

@@ -53,9 +53,10 @@ module MongoOplogBackup
 
       query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]
 
-      config.mongodump(['--out', config.oplog_dump_folder,
-        '--db', 'local', '--collection', 'oplog.rs'] +
-        query)
+      dump_args = ['--out', config.oplog_dump_folder, '--db', 'local', '--collection', 'oplog.rs']
+      dump_args += query
+      dump_args << '--gzip' if config.use_compression?
+      config.mongodump(dump_args)
 
       unless File.exists? config.oplog_dump
         raise "mongodump failed"
@@ -89,6 +90,7 @@ module MongoOplogBackup
         result[:empty] = true
       else
         outfile = "oplog-#{first}-#{last}.bson"
+        outfile += '.gz' if config.use_compression?
         full_path = File.join(backup_folder, outfile)
         FileUtils.mkdir_p backup_folder
         FileUtils.mv config.oplog_dump, full_path
@@ -124,7 +126,9 @@ module MongoOplogBackup
         raise "Backup folder '#{backup_folder}' already exists; not performing backup."
       end
       dump_folder = File.join(backup_folder, 'dump')
-      result = config.mongodump('--out', dump_folder)
+      dump_args = ['--out', dump_folder]
+      dump_args << '--gzip' if config.use_compression?
+      result = config.mongodump(dump_args)
       unless File.directory? dump_folder
         MongoOplogBackup.log.error 'Backup folder does not exist'
         raise 'Full backup failed'
@@ -159,7 +163,7 @@ module MongoOplogBackup
       end
 
       if mode == :oplog
-        raise "Unknown backup position - cannot perform oplog backup." unless have_backup
+        raise "Unknown backup position - cannot perform oplog backup. Have you completed a full backup?" unless have_backup
         MongoOplogBackup.log.info "Performing incremental oplog backup"
         lock(File.join(backup_folder, 'backup.lock')) do
           result = backup_oplog

data/lib/mongo_oplog_backup/config.rb

@@ -14,6 +14,7 @@ module MongoOplogBackup
       options = {}
       unless file.nil?
         conf = YAML.load_file(file)
+        options[:gzip] = conf["gzip"] unless conf["gzip"].nil?
         options[:ssl] = conf["ssl"] unless conf["ssl"].nil?
         options[:sslAllowInvalidCertificates] = conf["sslAllowInvalidCertificates"] unless conf["sslAllowInvalidCertificates"].nil?
         options[:sslCAFile] = conf["sslCAFile"] unless conf["sslCAFile"].nil?
@@ -30,6 +31,10 @@ module MongoOplogBackup
       options[:dir]
     end
 
+    def use_compression?
+      !!options[:gzip]
+    end
+
     def command_line_options
       args = []
       args << '--ssl' if options[:ssl]
@@ -48,7 +53,11 @@ module MongoOplogBackup
     end
 
     def oplog_dump
-      File.join(oplog_dump_folder, 'local/oplog.rs.bson')
+      if use_compression?
+        File.join(oplog_dump_folder, 'local/oplog.rs.bson.gz')
+      else
+        File.join(oplog_dump_folder, 'local/oplog.rs.bson')
+      end
     end
 
     def global_state_file

data/lib/mongo_oplog_backup/oplog.rb

@@ -1,11 +1,19 @@
+require 'zlib'
 module MongoOplogBackup
   module Oplog
     def self.each_document(filename)
-      File.open(filename, 'rb') do |stream|
+      yield_bson_document = Proc.new do |stream|
         while !stream.eof?
+          # FIXME: Since bson4, from_bson takes a ByteArray instead of a StringIO
           yield BSON::Document.from_bson(stream)
         end
       end
+
+      if gzip_fingerprint(filename)
+        Zlib::GzipReader.open(filename, &yield_bson_document)
+      else
+        File.open(filename, 'rb', &yield_bson_document)
+      end
     end
 
     def self.oplog_timestamps(filename)
@@ -18,7 +26,7 @@ module MongoOplogBackup
       timestamps
     end
 
-    FILENAME_RE = /\/oplog-(\d+):(\d+)-(\d+):(\d+)\.bson\z/
+    FILENAME_RE = /\/oplog-(\d+):(\d+)-(\d+):(\d+)\.bson(?:\.gz)?\z/
 
     def self.timestamps_from_filename filename
       match = FILENAME_RE.match(filename)
@@ -38,8 +46,9 @@ module MongoOplogBackup
     def self.merge(target, source_files, options={})
       limit = options[:limit] # TODO: use
       force = options[:force]
+      compress = !!options[:gzip]
 
-      File.open(target, 'wb') do |output|
+      process_output = Proc.new do |output|
         last_timestamp = nil
         first = true
 
@@ -65,6 +74,10 @@ module MongoOplogBackup
           Oplog.each_document(filename) do |doc|
             timestamp = doc['ts']
             first_file_timestamp = timestamp if first_file_timestamp.nil?
+
+            # gzip stores the mtime in the header, so we set it explicity for consistency between runs.
+            output.mtime = first_file_timestamp.seconds if output.mtime.to_i == 0
+
             if !last_timestamp.nil? && timestamp <= last_timestamp
               skipped += 1
             elsif !last_file_timestamp.nil? && timestamp <= last_file_timestamp
@@ -90,10 +103,15 @@ module MongoOplogBackup
           first = false
         end
       end
+      if (compress)
+        Zlib::GzipWriter.open(target, &process_output)
+      else
+        File.open(target, 'wb', &process_output)
+      end
     end
 
     def self.find_oplogs(dir)
-      files = Dir.glob(File.join(dir, 'oplog-*.bson'))
+      files = Dir.glob(File.join(dir, 'oplog-*.bson*'))
       files.keep_if {|name| name =~ FILENAME_RE}
       files.sort! {|a, b| timestamps_from_filename(a)[:first] <=> timestamps_from_filename(b)[:first]}
       files
@@ -101,9 +119,15 @@ module MongoOplogBackup
 
     def self.merge_backup(dir)
       oplogs = find_oplogs(dir)
-      target = File.join(dir, 'dump', 'oplog.bson')
+      compress_target = oplogs.any? { |o| o.end_with?('.gz') }
+      target = File.join(dir, 'dump', 'oplog.bson') # Mongorestore expects this filename, without a gzip suffix.
       FileUtils.mkdir_p(File.join(dir, 'dump'))
-      merge(target, oplogs)
+      merge(target, oplogs, {gzip: compress_target})
+    end
+
+    def self.gzip_fingerprint filename
+      bytes = File.read(filename, 2, 0)
+      bytes[0] == "\x1f".force_encoding('BINARY') && bytes[1] == "\x8b".force_encoding('BINARY')
     end
 
   end

data/lib/mongo_oplog_backup/version.rb

@@ -1,3 +1,3 @@
 module MongoOplogBackup
-  VERSION = "0.0.9"
+  VERSION = "0.0.10"
 end

data/spec/oplog_spec.rb

@@ -55,4 +55,58 @@ describe MongoOplogBackup::Oplog do
 
     'spec-tmp/backup/dump/oplog.bson'.should be_same_oplog_as oplog_merged
   end
+
+
+
+  context 'with gzipped oplogs' do
+    let(:oplog1) { 'spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz'}
+    let(:oplog2) { 'spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz'}
+    let(:oplog3) { 'spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz'}
+    let(:oplog_merged) { 'spec/fixtures/gzip/oplog-merged-gzipped.bson'}
+
+    it 'should extract oplog timestamps' do
+      timestamps = MongoOplogBackup::Oplog.oplog_timestamps(oplog1)
+      timestamps.should == [
+        BSON::Timestamp.new(1479827504, 7),
+        BSON::Timestamp.new(1479827515, 1),
+        BSON::Timestamp.new(1479827517, 1),
+        BSON::Timestamp.new(1479827518, 1)
+      ]
+    end
+
+    it 'should merge oplogs' do
+      merged_out = 'spec-tmp/oplog-merged-gzipped.bson'
+      MongoOplogBackup::Oplog.merge(merged_out, [oplog1, oplog2, oplog3], {gzip: true})
+
+      expected_timestamps =
+        MongoOplogBackup::Oplog.oplog_timestamps(oplog1) +
+        MongoOplogBackup::Oplog.oplog_timestamps(oplog2) +
+        MongoOplogBackup::Oplog.oplog_timestamps(oplog3)
+
+      expected_timestamps.uniq!
+      expected_timestamps.sort!  # Not sure if uniq! modifies the order
+
+      actual_timestamps = MongoOplogBackup::Oplog.oplog_timestamps(merged_out)
+      actual_timestamps.should == expected_timestamps
+
+      merged_out.should be_same_oplog_as oplog_merged
+    end
+
+    it 'should parse timestamps from a filename' do
+      timestamps = MongoOplogBackup::Oplog.timestamps_from_filename('some/oplog-1408088734:1-1408088740:52.bson.gz')
+      timestamps.should == {
+        first: BSON::Timestamp.new(1408088734, 1),
+        last: BSON::Timestamp.new(1408088740, 52)
+      }
+    end
+
+    it "should merge a backup folder" do
+      FileUtils.mkdir_p 'spec-tmp/backup-zipped'
+      FileUtils.cp_r Dir['spec/fixtures/gzip/oplog-*.bson.gz'], 'spec-tmp/backup-zipped/'
+
+      MongoOplogBackup::Oplog.merge_backup('spec-tmp/backup-zipped')
+
+      'spec-tmp/backup-zipped/dump/oplog.bson'.should be_same_oplog_as oplog_merged
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mongo-oplog-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Ralf Kistner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-12 00:00:00.000000000 Z
+date: 2016-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson
@@ -125,6 +125,10 @@ files:
 - spec/backup_spec.rb
 - spec/command_spec.rb
 - spec/enumerable_spec.rb
+- spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz
+- spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz
+- spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz
+- spec/fixtures/gzip/oplog-merged-gzipped.bson
 - spec/fixtures/oplog-1408088734:1-1408088740:1.bson
 - spec/fixtures/oplog-1408088740:1-1408088810:1.bson
 - spec/fixtures/oplog-1408088810:1-1408088928:1.bson
@@ -160,6 +164,10 @@ test_files:
 - spec/backup_spec.rb
 - spec/command_spec.rb
 - spec/enumerable_spec.rb
+- spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz
+- spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz
+- spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz
+- spec/fixtures/gzip/oplog-merged-gzipped.bson
 - spec/fixtures/oplog-1408088734:1-1408088740:1.bson
 - spec/fixtures/oplog-1408088740:1-1408088810:1.bson
 - spec/fixtures/oplog-1408088810:1-1408088928:1.bson