mongo-oplog-backup 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3acf779f860fdb9a7fc941b68f7755c36f215ad6
-  data.tar.gz: 5a1a19a402c4d337b90c81efc693aeb7d583e454
+  metadata.gz: 0683856bbe55049a12b2d1116dab286a2777ac0a
+  data.tar.gz: ec2de886b2be3a374fa7b19e5882dc0ba81d2235
 SHA512:
-  metadata.gz: 5c01fe3144e8a2c891c363bac256082c17f836d71fa2b097d8f3e737afcaba51d41034b6c7dda7025ac5d0ee282a35908796441bbb9bf41cafc0aa89ac9e35af
-  data.tar.gz: 94b42dac6b402b476c83ff6f1f131884d266d2b8bc545b389440574fc36f084bcbd7df9152d6149ee1d2f9a2c37e47864822d795c0d619109c93a3de2d1a17e1
+  metadata.gz: 9afd48be80dc546dfd14c6486eb4a324f5cb4c42deb7eda7d95c5f51e10827f2afb7f2cd19876d7bde06af8feac632c211e2e21da3c8b40305f7291b359f0cab
+  data.tar.gz: fe30271a6df011ebe714408f4311020c6c9d339e8bfb94ed7a27ef941c933437c46de2212a8562afd272675040d18708b59869d062bd635cf33bcc21a3350691

data/README.md

@@ -2,7 +2,7 @@
 
 **Experimental** incremental backup system for MongoDB based on the oplog.
 
-Not ready for any important data yet.
+**Not ready for any important data yet. Use at your own risk.**
 
 ## Installation
 
@@ -12,9 +12,24 @@ Not ready for any important data yet.
 
 ## Usage
 
+To backup from localhost to the `mybackup` directory.
+
     mongo-oplog-backup backup --dir mybackup
 
-TODO: Write usage instructions here
+The first run will perform a full backup. Subsequent runs will backup any new entries from the oplog.
+A full backup can be forced with the `--full` option.
+
+For connection options, see `mongo-oplog-backup backup --help`.
+
+## To restore
+
+    mongo-oplog-backup merge --dir mybackup/backup-<timestamp>
+    
+The above command merges the individual oplog backups into `mybackup/backup-<timestamp>/dump/oplog.bson`.
+This allows you to restore the backup with the `mongorestore` command:
+
+    mongorestore --drop --oplogReplay backup/backup-<timestamp>/dump
+    
 
 ## Backup structure
 
@@ -25,9 +40,10 @@ TODO: Write usage instructions here
   * `oplog-<start>-<end>.bson` - The oplog from the start timestamp until the end timestamp (inclusive).
 
 Each time a full backup is performed, a new backup folder is created.
+
 ## Contributing
 
-1. Fork it ( http://github.com/<my-github-username>/mongo-oplog-backup/fork )
+1. Fork it ( http://github.com/journeyapps/mongo-oplog-backup/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/bin/mongo-oplog-backup

@@ -12,7 +12,9 @@ opts = Slop.parse(help: true, strict: true) do
     on :d, :dir, "Directory to store backup files. Defaults to 'backup'.", argument: :required
     on :full, 'Force full backup'
     on :oplog, 'Force oplog backup'
+    on :'if-not-busy', 'Do nothing when another backup is busy running.'
 
+    on :f, :file, 'Configuration file for common defaults', argument: :required
     on :ssl, "Connect to a mongod instance over an SSL connection"
     on :host, "Specifies a resolvable hostname for the mongod that you wish to backup.", default: 'localhost', argument: :required
     on :port, "Specifies the port that mongod is running on", default: '27017', argument: :required
@@ -25,6 +27,7 @@ opts = Slop.parse(help: true, strict: true) do
         dir: dir,
         ssl: opts.ssl?
       }
+      config_opts[:file] = opts[:file]
       config_opts[:host] = opts[:host]
       config_opts[:port] = opts[:port]
       config_opts[:username] = opts[:username]
@@ -38,7 +41,7 @@ opts = Slop.parse(help: true, strict: true) do
       end
       config = MongoOplogBackup::Config.new(config_opts)
       backup = MongoOplogBackup::Backup.new(config)
-      backup.perform(mode)
+      backup.perform(mode, if_not_busy: opts.if_not_busy?)
     end
   end
 

data/lib/mongo_oplog_backup/backup.rb

@@ -2,6 +2,9 @@ require 'json'
 require 'fileutils'
 require 'mongo_oplog_backup/oplog'
 
+class LockError < StandardError
+end
+
 module MongoOplogBackup
   class Backup
     attr_reader :config
@@ -10,6 +13,16 @@ module MongoOplogBackup
       @config = config
     end
 
+    def lock(lockname, &block)
+      File.open(lockname, File::RDWR|File::CREAT, 0644) do |file|
+        got_lock = file.flock(File::LOCK_EX|File::LOCK_NB)
+        if got_lock == false
+          raise LockError, "Failed to acquire lock - another backup may be busy"
+        end
+        yield
+      end
+    end
+
     def backup_oplog(options={})
       start_at = options[:start]
       backup = options[:backup]
@@ -91,42 +104,61 @@ module MongoOplogBackup
       }
     end
 
-    def perform(mode=:auto)
-      state_file = config.state_file
-      state = JSON.parse(File.read(state_file)) rescue nil
-      state ||= {}
-      have_position = (state['position'] && state['backup'])
-
-      if mode == :auto
-        if have_position
-          mode = :oplog
-        else
-          mode = :full
+    def perform(mode=:auto, options={})
+      if_not_busy = options[:if_not_busy] || false
+
+      perform_oplog_afterwards = false
+
+      lock(config.lock_file) do
+        state_file = config.state_file
+        state = JSON.parse(File.read(state_file)) rescue nil
+        state ||= {}
+        have_position = (state['position'] && state['backup'])
+
+        if mode == :auto
+          if have_position
+            mode = :oplog
+          else
+            mode = :full
+          end
         end
-      end
 
-      if mode == :oplog
-        raise "Unknown backup position - cannot perform oplog backup." unless have_position
-        MongoOplogBackup.log.info "Performing incremental oplog backup"
-        position = BSON::Timestamp.from_json(state['position'])
-        result = backup_oplog(start: position, backup: state['backup'])
-        unless result[:empty]
-          new_entries = result[:entries] - 1
-          state['position'] = result[:position]
+        if mode == :oplog
+          raise "Unknown backup position - cannot perform oplog backup." unless have_position
+          MongoOplogBackup.log.info "Performing incremental oplog backup"
+          position = BSON::Timestamp.from_json(state['position'])
+          result = backup_oplog(start: position, backup: state['backup'])
+          unless result[:empty]
+            new_entries = result[:entries] - 1
+            state['position'] = result[:position]
+            File.write(state_file, state.to_json)
+            MongoOplogBackup.log.info "Backed up #{new_entries} new entries to #{result[:file]}"
+          else
+            MongoOplogBackup.log.info "Nothing new to backup"
+          end
+        elsif mode == :full
+          MongoOplogBackup.log.info "Performing full backup"
+          result = backup_full
+          state = result
           File.write(state_file, state.to_json)
-          MongoOplogBackup.log.info "Backed up #{new_entries} new entries to #{result[:file]}"
-        else
-          MongoOplogBackup.log.info "Nothing new to backup"
+          MongoOplogBackup.log.info "Performed full backup"
+
+          perform_oplog_afterwards = true
         end
-      elsif mode == :full
-        MongoOplogBackup.log.info "Performing full backup"
-        result = backup_full
-        state = result
-        File.write(state_file, state.to_json)
-        MongoOplogBackup.log.info "Performed full backup"
+      end
 
+      # Has to be outside the lock
+      if perform_oplog_afterwards
         # Oplog backup
-        perform(:oplog)
+        perform(:oplog, options)
+      end
+
+    rescue LockError => e
+      if if_not_busy
+        MongoOplogBackup.log.info e.message
+        MongoOplogBackup.log.info 'Not performing backup'
+      else
+        raise
       end
     end
 

data/lib/mongo_oplog_backup/config.rb

@@ -3,7 +3,23 @@ module MongoOplogBackup
     attr_reader :options
 
     def initialize(options)
-      @options = options
+      config_file = options.delete(:file)
+      # Command line options take precedence
+      @options = from_file(config_file).merge(options) 
+    end
+
+    def from_file file
+      options = {}
+      unless file.nil?
+        conf = YAML.load_file(file)
+        options[:ssl] = conf["ssl"] unless conf["ssl"].nil?
+        options[:host] = conf["host"] unless conf["host"].nil?
+        options[:port] = conf["port"].to_s unless conf["port"].nil?
+        options[:username] = conf["username"] unless conf["username"].nil?
+        options[:password] = conf["password"] unless conf["password"].nil?
+      end
+      
+      options
     end
 
     def backup_dir
@@ -31,6 +47,10 @@ module MongoOplogBackup
       File.join(backup_dir, 'backup.json')
     end
 
+    def lock_file
+      File.join(backup_dir, 'backup.lock')
+    end
+
     def exec(cmd)
       MongoOplogBackup.log.debug ">>> #{cmd}"
       `#{cmd}`

data/lib/mongo_oplog_backup/version.rb

@@ -1,3 +1,3 @@
 module MongoOplogBackup
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/sample-config.yml

@@ -0,0 +1,6 @@
+# Any settings specified here are overridden by command line options.
+host: localhost
+port: 27017
+username: backup-user
+password: s3kr1t
+ssl: true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mongo-oplog-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Ralf Kistner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-15 00:00:00.000000000 Z
+date: 2014-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson
@@ -120,6 +120,7 @@ files:
 - lib/mongo_oplog_backup/version.rb
 - mongo-oplog-backup.gemspec
 - oplog-last-timestamp.js
+- sample-config.yml
 - spec/backup_spec.rb
 - spec/enumerable_spec.rb
 - spec/fixtures/oplog-1408088734:1-1408088740:1.bson