mongo-oplog-backup 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58c9a1ccd7bbe033e5ebd295015a4b0f092d9525
-  data.tar.gz: d7a0ccf762945fc2ffafbbd3f24955397150928e
+  metadata.gz: 41408b43d702bfbe7c3b4ea5ff0d46c89893d6e9
+  data.tar.gz: cb54777ed47f5c31f1bd9aef664397b59f19806b
 SHA512:
-  metadata.gz: 8f62c0d4cdeac23d1e223424f5d5a930c81d4317c8acb5282d1e44b4a3ddb6453a4c7d0627f72b09c22f38f0d505cb0c6dcf89e10aae2b9bae465689ba4d6a55
-  data.tar.gz: 4dcbd7c51dc11cb87b0cd9e20a7726ee5e629b7692c330037d1ae5b027d59268ee0afb9b19b18a866163bbf7eadad4f77d8eca4766ea64aa803fad0539fc5462
+  metadata.gz: ec83326fefad756510b66faf9fd8b88e268f140ba447c803bc3898cc1345565c9d58aaab71877916d3272bf69e797bcdbd66ee34b6d2bf8c25fdd066aeebde46
+  data.tar.gz: 7674468a302f77ce3b0039b9e53d6182c09f810542b4b76cb7c14a302f7c08051de6d53d9cfc67a8144bc67e657c92c8d7064dbb2e7c233dc41fab59dc46df01

data/README.md

@@ -4,8 +4,39 @@
 
 **Not ready for any important data yet. Use at your own risk.**
 
+## Introduction
+
+This project aims to enable incremental backups with point-in-time restore
+functionality, utilizing MongoDB's oplog and standard tools wherever possible.
+
+A backup script can be run from a cron job, and each incremental run produces
+a single file that can be stored on your preferred medium, for example Amazon S3
+or an FTP site. This project only provides the tools to produce the backup files,
+and it's up to you to transfer it to a backup medium.
+
+Interally the `mongodump` command is used for the backup operations. Initially
+a full dump is performed, after which incremetal backups are performed by backing
+up new sections of the oplog. Only the standard BSON format from mongodump is used.
+
+To restore a backup, the incremental oplogs are merged into a single file and combined
+with the initial full dump, which can then be restored with a standard
+`mongorestore --oplogReplay` command. A point-in-time restore with the `--oplogLimit`
+option of `mongorestore`. Additional support for this may be added to the
+oplog merging command in the future to simplify the process.
+
+Incremental oplogs always overlap by exactly one entry, so that integrity can easily
+be verified (e.g. that there are no gaps between incremental oplogs).
+
+
+
 ## Installation
 
+Install released gem (recommended):
+
+    gem install mongo-oplog-backup
+
+Install latest development version:
+
     git clone git@github.com:journeyapps/mongo-oplog-backup.git
     cd mongo-oplog-backup
     rake install
@@ -19,23 +50,39 @@ To backup from localhost to the `mybackup` directory.
 The first run will perform a full backup. Subsequent runs will backup any new entries from the oplog.
 A full backup can be forced with the `--full` option.
 
-For connection options, see `mongo-oplog-backup backup --help`.
+Sample cron script to perform incremental backups every 15 minutes, and a full backup once a week at 00:05:
+
+    0,15,30,45 * * * * /path/to/ruby/bin/mongo-oplog-backup backup --dir /path/to/backup/location --oplog >> /path/to/backup.log
+    5 0 * * 1 /path/to/ruby/bin/mongo-oplog-backup backup --dir /path/to/backup/location --full >> /path/to/backup.log
+
+It is recommended to do a full backup every few days. The restore process may
+be very inefficient if the oplogs grow larger than a full backup.
+
+For connection and authentication options, see `mongo-oplog-backup backup --help`.
+
+The backup commands work on a live server. The initial dump with oplog replay relies
+on the idempotency of the oplog to have a consistent snapshot, similar to `mongodump --oplog`.
+That said, there have been bugs in the past that caused the oplog to not be idempotent
+in some edge cases. Therefore it is recommended to stop the secondary before performing
+a full backup.
 
 ## To restore
 
     mongo-oplog-backup merge --dir mybackup/backup-<timestamp>
-    
+
 The above command merges the individual oplog backups into `mybackup/backup-<timestamp>/dump/oplog.bson`.
 This allows you to restore the backup with the `mongorestore` command:
 
     mongorestore --drop --oplogReplay backup/backup-<timestamp>/dump
-    
 
 ## Backup structure
 
 * `backup.json` - Stores the current state (oplog timestamp and backup folder).
     The only file required to perform incremental backups. It is not used for restoring a backup.
+* `backup.lock` - Lock file to prevent two full backups from running concurrently.
 * `backup-<timestamp>` - The current backup folder.
+  * `backup.lock` - Lock file preventing two backups running concurrently in this folder.
+  * `status.json` - backup status (oplog timestamp)
   * `dump` - a full mongodump
   * `oplog-<start>-<end>.bson` - The oplog from the start timestamp until the end timestamp (inclusive).
 

data/bin/mongo-oplog-backup

@@ -12,7 +12,6 @@ opts = Slop.parse(help: true, strict: true) do
     on :d, :dir, "Directory to store backup files. Defaults to 'backup'.", argument: :required
     on :full, 'Force full backup'
     on :oplog, 'Force oplog backup'
-    on :'if-not-busy', 'Do nothing when another backup is busy running.'
 
     on :f, :file, 'Configuration file for common defaults', argument: :required
     on :ssl, "Connect to a mongod instance over an SSL connection"
@@ -41,7 +40,7 @@ opts = Slop.parse(help: true, strict: true) do
       end
       config = MongoOplogBackup::Config.new(config_opts)
       backup = MongoOplogBackup::Backup.new(config)
-      backup.perform(mode, if_not_busy: opts.if_not_busy?)
+      backup.perform(mode)
     end
   end
 

data/lib/mongo_oplog_backup/backup.rb

@@ -7,14 +7,35 @@ end
 
 module MongoOplogBackup
   class Backup
-    attr_reader :config
+    attr_reader :config, :backup_name
 
-    def initialize(config)
+    def backup_folder
+      return nil unless backup_name
+      File.join(config.backup_dir, backup_name)
+    end
+
+    def state_file
+      File.join(backup_folder, 'state.json')
+    end
+
+    def initialize(config, backup_name=nil)
       @config = config
+      @backup_name = backup_name
+      if backup_name.nil?
+        state_file = config.global_state_file
+        state = JSON.parse(File.read(state_file)) rescue nil
+        state ||= {}
+        @backup_name = state['backup']
+      end
+    end
+
+    def write_state(state)
+      File.write(state_file, state.to_json)
     end
 
     def lock(lockname, &block)
       File.open(lockname, File::RDWR|File::CREAT, 0644) do |file|
+        # Get a non-blocking lock
         got_lock = file.flock(File::LOCK_EX|File::LOCK_NB)
         if got_lock == false
           raise LockError, "Failed to acquire lock - another backup may be busy"
@@ -24,16 +45,14 @@ module MongoOplogBackup
     end
 
     def backup_oplog(options={})
-      start_at = options[:start]
-      backup = options[:backup]
-      raise ArgumentError, ":backup is required" unless backup
+      raise ArgumentError, "No state in #{backup_name}" unless File.exists? state_file
+
+      backup_state = JSON.parse(File.read(state_file))
+      start_at = options[:start] || BSON::Timestamp.from_json(backup_state['position'])
       raise ArgumentError, ":start is required" unless start_at
 
-      if start_at
-        query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]
-      else
-        query = []
-      end
+      query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]
+
       config.mongodump(['--out', config.oplog_dump_folder,
         '--db', 'local', '--collection', 'oplog.rs'] +
         query)
@@ -70,10 +89,13 @@ module MongoOplogBackup
         result[:empty] = true
       else
         outfile = "oplog-#{first}-#{last}.bson"
-        full_path = File.join(config.backup_dir, backup, outfile)
-        FileUtils.mkdir_p File.join(config.backup_dir, backup)
+        full_path = File.join(backup_folder, outfile)
+        FileUtils.mkdir_p backup_folder
         FileUtils.mv config.oplog_dump, full_path
 
+        write_state({
+          'position' => result[:position]
+        })
         result[:file] = full_path
         result[:empty] = false
       end
@@ -97,12 +119,27 @@ module MongoOplogBackup
     def backup_full
       position = latest_oplog_timestamp
       raise "Cannot backup with empty oplog" if position.nil?
-      backup_name = "backup-#{position}"
-      dump_folder = File.join(config.backup_dir, backup_name, 'dump')
-      config.mongodump('--out', dump_folder)
+      @backup_name = "backup-#{position}"
+      if File.exists? backup_folder
+        raise "Backup folder '#{backup_folder}' already exists; not performing backup."
+      end
+      dump_folder = File.join(backup_folder, 'dump')
+      result = config.mongodump('--out', dump_folder)
       unless File.directory? dump_folder
+        MongoOplogBackup.log.error 'Backup folder does not exist'
         raise 'Full backup failed'
       end
+
+      File.write(File.join(dump_folder, 'debug.log'), result.standard_output)
+
+      unless result.standard_error.length == 0
+        File.write(File.join(dump_folder, 'error.log'), result.standard_error)
+      end
+
+      write_state({
+        'position' => position
+      })
+
       return {
         position: position,
         backup: backup_name
@@ -110,62 +147,40 @@ module MongoOplogBackup
     end
 
     def perform(mode=:auto, options={})
-      if_not_busy = options[:if_not_busy] || false
-
-      perform_oplog_afterwards = false
-
       FileUtils.mkdir_p config.backup_dir
-      lock(config.lock_file) do
-        state_file = config.state_file
-        state = JSON.parse(File.read(state_file)) rescue nil
-        state ||= {}
-        have_position = (state['position'] && state['backup'])
+      have_backup = backup_folder != nil
 
-        if mode == :auto
-          if have_position
-            mode = :oplog
-          else
-            mode = :full
-          end
+      if mode == :auto
+        if have_backup
+          mode = :oplog
+        else
+          mode = :full
         end
+      end
 
-        if mode == :oplog
-          raise "Unknown backup position - cannot perform oplog backup." unless have_position
-          MongoOplogBackup.log.info "Performing incremental oplog backup"
-          position = BSON::Timestamp.from_json(state['position'])
-          result = backup_oplog(start: position, backup: state['backup'])
+      if mode == :oplog
+        raise "Unknown backup position - cannot perform oplog backup." unless have_backup
+        MongoOplogBackup.log.info "Performing incremental oplog backup"
+        lock(File.join(backup_folder, 'backup.lock')) do
+          result = backup_oplog
           unless result[:empty]
             new_entries = result[:entries] - 1
-            state['position'] = result[:position]
-            File.write(state_file, state.to_json)
             MongoOplogBackup.log.info "Backed up #{new_entries} new entries to #{result[:file]}"
           else
             MongoOplogBackup.log.info "Nothing new to backup"
           end
-        elsif mode == :full
+        end
+      elsif mode == :full
+        lock(config.global_lock_file) do
           MongoOplogBackup.log.info "Performing full backup"
           result = backup_full
-          state = result
-          File.write(state_file, state.to_json)
+          File.write(config.global_state_file, {
+            'backup' => result[:backup]
+          }.to_json)
           MongoOplogBackup.log.info "Performed full backup"
-
-          perform_oplog_afterwards = true
         end
-      end
-
-      # Has to be outside the lock
-      if perform_oplog_afterwards
-        # Oplog backup
         perform(:oplog, options)
       end
-
-    rescue LockError => e
-      if if_not_busy
-        MongoOplogBackup.log.info e.message
-        MongoOplogBackup.log.info 'Not performing backup'
-      else
-        raise
-      end
     end
 
     def latest_oplog_timestamp_moped

data/lib/mongo_oplog_backup/config.rb

@@ -41,18 +41,18 @@ module MongoOplogBackup
     end
 
     def oplog_dump_folder
-      File.join(backup_dir, 'dump')
+      File.join(backup_dir, 'tmp-dump')
     end
 
     def oplog_dump
       File.join(oplog_dump_folder, 'local/oplog.rs.bson')
     end
 
-    def state_file
+    def global_state_file
       File.join(backup_dir, 'backup.json')
     end
 
-    def lock_file
+    def global_lock_file
       File.join(backup_dir, 'backup.lock')
     end
 

data/lib/mongo_oplog_backup/version.rb

@@ -1,3 +1,3 @@
 module MongoOplogBackup
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 end

data/spec/backup_spec.rb

@@ -6,7 +6,7 @@ describe MongoOplogBackup do
     MongoOplogBackup::VERSION.should_not be_nil
   end
 
-  let(:backup) { MongoOplogBackup::Backup.new(MongoOplogBackup::Config.new dir: 'spec-tmp/backup') }
+  let(:backup) { MongoOplogBackup::Backup.new(MongoOplogBackup::Config.new(dir: 'spec-tmp/backup'), 'backup1') }
 
   before(:all) do
     # We need one entry in the oplog to start with
@@ -44,12 +44,18 @@ describe MongoOplogBackup do
       end
     end
     last = backup.latest_oplog_timestamp
-    result = backup.backup_oplog(start: first, backup: 'backup1')
+    FileUtils.mkdir_p backup.backup_folder
+    backup.write_state({position: first})
+    result = backup.backup_oplog(backup: 'backup1')
+    result[:entries].should == 6
+    result[:empty].should == false
+    result[:position].should == last
+    result[:first].should == first
+
     file = result[:file]
     timestamps = MongoOplogBackup::Oplog.oplog_timestamps(file)
     timestamps.count.should == 6
     timestamps.first.should == first
     timestamps.last.should == last
-
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mongo-oplog-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Ralf Kistner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-22 00:00:00.000000000 Z
+date: 2014-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson