mongo-oplog-backup 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c98c007c62b80bc6c87b1f141deba9d27933865
-  data.tar.gz: 9a731f45965d4218bbef98e711908d236d41d29c
+  metadata.gz: 614d931c8a55e33ca69b78eb20c38204c2668d50
+  data.tar.gz: 5807c75c2173be443b36ceee09ca5e38c545a4c6
 SHA512:
-  metadata.gz: 5642b1a2790b9d2493c1d5e7f90e8baae53bc6d9c6a81084475d03100903b6cb704124f7a4336060f7c9256900db00a677b06c548c665f5ccdcbae7989f28bea
-  data.tar.gz: cbba467fb5f8c9ffe9c30a79298c6d3779d5e590ebab20bf874139b76b1adb288560caf7910f734093f0dfe5ce5e563a041c8b7ce5bc4af0f75ae8f8d3a3aef6
+  metadata.gz: dd246853e1c69ceac5a503d6bf9a684c9bc6ab44debdffaa0ced2389521194cdaf9751b9c72f61dd96b912fe2f3857a19c9c4e99b979975f51443ace77e4fd74
+  data.tar.gz: 7713c2c5de857b47eaaddd6b8304bad7d7c97f56aa176ba71abfddcba9159de003f1b5c72718de2ff825c350aaea15bed215a6401dfa3de683afcc2be135e381

data/README.md

@@ -66,6 +66,19 @@ That said, there have been bugs in the past that caused the oplog to not be idem
 in some edge cases. Therefore it is recommended to stop the secondary before performing
 a full backup.
 
+## Rotation
+
+Older backups can be automatically deleted with `mongo-oplog-backup rotate`.
+
+For example, given the full backup schedule above and a recovery point objective of 4 weeks, any full backup sets
+older than 4 weeks (excluding the currently active backup set and the previous backup set) can be deleted.
+
+A sample cron script to clean-up any older backups about an hour before the weekly full backup:
+
+   0 23 * * 0 /path/to/ruby/bin/mongo-oplog-backup rotate --dir /path/to/backup/location --keep_days=28 >> /path/to/backup.log
+   
+It is strongly recommended to use the `dryRun` option to thoroughly test your particular configuration and schedule. 
+
 ## To restore
 
     mongo-oplog-backup merge --dir mybackup/backup-<timestamp>

data/bin/mongo-oplog-backup

@@ -73,6 +73,26 @@ opts = Slop.parse(help: true, strict: true) do
     end
   end
 
+  command 'rotate' do
+  banner "Delete old backup sets.\nUsage: mongo-oplog-backup rotate [options]"
+    on :v, :verbose, "Enable verbose mode"
+    on :d, :dir, "Directory where backup files are stored. Defaults to 'backup'.", argument: :required
+    on :keepDays, "Minimum number of days of backups to keep (recovery point objective). This should be AT LEAST equal to the Full backup schedule.", default: '32', argument: :required
+    on :dryRun, "Perform a trial run with no changes made."
+
+    run do |opts, args|
+      dir = opts[:dir] || 'backup'
+      config_opts = {
+          dir: dir,
+          keepDays: (opts[:keepDays].to_i rescue nil),
+          dryRun: opts.dryRun?
+      }
+      config = MongoOplogBackup::Config.new(config_opts)
+      rotate = MongoOplogBackup::Rotate.new(config)
+      rotate.perform
+    end
+  end
+
   command 'restore' do
   banner 'Usage: mongo-oplog-backup restore [options]'
     on :v, :verbose, 'Enable verbose mode'

data/lib/mongo_oplog_backup.rb

@@ -4,11 +4,13 @@ require 'mongo_oplog_backup/version'
 require 'mongo_oplog_backup/ext/enumerable'
 require 'mongo_oplog_backup/ext/timestamp'
 
+require 'mongo_oplog_backup/lock'
 require 'mongo_oplog_backup/command'
 require 'mongo_oplog_backup/config'
 require 'mongo_oplog_backup/backup'
 require 'mongo_oplog_backup/oplog'
 require 'mongo_oplog_backup/restore'
+require 'mongo_oplog_backup/rotate'
 
 module MongoOplogBackup
   def self.log

data/lib/mongo_oplog_backup/backup.rb

@@ -2,11 +2,9 @@ require 'json'
 require 'fileutils'
 require 'mongo_oplog_backup/oplog'
 
-class LockError < StandardError
-end
-
 module MongoOplogBackup
   class Backup
+    include Lockable
     attr_reader :config, :backup_name
 
     def backup_folder
@@ -33,17 +31,6 @@ module MongoOplogBackup
       File.write(state_file, state.to_json)
     end
 
-    def lock(lockname, &block)
-      File.open(lockname, File::RDWR|File::CREAT, 0644) do |file|
-        # Get a non-blocking lock
-        got_lock = file.flock(File::LOCK_EX|File::LOCK_NB)
-        if got_lock == false
-          raise LockError, "Failed to acquire lock - another backup may be busy"
-        end
-        yield
-      end
-    end
-
     def backup_oplog(options={})
       raise ArgumentError, "No state in #{backup_name}" unless File.exists? state_file
 

data/lib/mongo_oplog_backup/lock.rb

@@ -0,0 +1,21 @@
+module MongoOplogBackup
+  class LockError < StandardError
+  end
+
+  module Lockable
+    def self.included(base)
+      base.extend(self)
+    end
+
+    def lock(lockname, &block)
+      File.open(lockname, File::RDWR|File::CREAT, 0644) do |file|
+        # Get a non-blocking lock
+        got_lock = file.flock(File::LOCK_EX|File::LOCK_NB)
+        if got_lock == false
+          raise LockError, "Failed to acquire lock - another backup may be busy"
+        end
+        yield
+      end
+    end
+  end
+end

data/lib/mongo_oplog_backup/rotate.rb

@@ -0,0 +1,90 @@
+require 'pathname'
+
+module MongoOplogBackup
+  class Rotate
+    include Lockable
+    attr_reader :config, :backup_list
+    DAY = 86400
+    RECOVERY_POINT_OBJECTIVE = 32 * DAY # Longest month + 1
+    KEEP_MINIMUM_SETS = 2 # Current & Previous
+
+    BACKUP_DIR_NAME_FORMAT = /\Abackup-\d+:\d+\z/
+
+    def initialize(config)
+      @config = config
+      @dry_run = !!@config.options[:dryRun]
+      @backup_list = find_backup_directories
+      if @config.options[:keepDays].nil?
+        @recovery_point_objective = RECOVERY_POINT_OBJECTIVE
+      else
+        @recovery_point_objective = @config.options[:keepDays] * DAY
+      end
+    end
+
+    def current_backup_name
+      if @current_backup_name.nil?
+        state_file = config.global_state_file
+        state = JSON.parse(File.read(state_file)) rescue nil
+        state ||= {}
+        @current_backup_name = state['backup']
+      end
+      @current_backup_name
+    end
+
+    def perform
+      lock(config.global_lock_file) do
+        MongoOplogBackup.log.info "Rotating out old backups."
+
+        if @backup_list.count >= 0 && @backup_list.count <= KEEP_MINIMUM_SETS
+          MongoOplogBackup.log.info "Too few backup sets to automatically rotate."
+        elsif @backup_list.count > KEEP_MINIMUM_SETS
+          filter_for_deletion(@backup_list).each do |path|
+            MongoOplogBackup.log.info "#{@dry_run ? '[DRYRUN] Would delete' : 'Deleting'} #{path}."
+            begin
+              FileUtils.remove_entry_secure(path) unless @dry_run
+            rescue StandardError => e
+              MongoOplogBackup.log.error "Delete failed: #{e.message}"
+            end
+          end
+        end
+
+        MongoOplogBackup.log.info "Rotating out old backups completed."
+      end
+    end
+
+    # Lists subdirectories in the backup location that match the backup set naming format and appear to have completed
+    # successfully.
+    # @return [Array<Pathname>] backup directories.
+    def find_backup_directories
+      dirs = Pathname.new(@config.backup_dir).children.select(&:directory?)
+      dirs = dirs.select { |dir| dir.basename.to_s =~ BACKUP_DIR_NAME_FORMAT }
+      dirs = dirs.select { |dir| File.exist?(File.join(dir, 'state.json')) }
+      dirs.sort
+    end
+
+    # @param [Array<Pathname>] source_list List of Pathnames for the full backup sets in the backup directory.
+    # @return [Array<Pathname]
+    def filter_for_deletion(source_list)
+      source_list = source_list.sort.reverse.drop(KEEP_MINIMUM_SETS) # Keep a minimum number of full backups
+      # The most recent dir might not be the active one (eg. if the mongodump fails). Ensure that it is excluded.
+      source_list = source_list.reject { |path| path.basename.to_s == current_backup_name }
+
+      source_list.select {|path| age_of_backup_in_seconds(path.basename) > @recovery_point_objective }
+    end
+
+    private
+
+    def age_of_backup_in_seconds(path)
+      Time.now.to_i - timestamp_from_string(path.to_s).seconds
+    end
+
+    # Accepts: <seconds>[:ordinal]
+    def timestamp_from_string(string)
+      match = /(\d+)(?::(\d+))?/.match(string)
+      return nil unless match
+      s1 = match[1].to_i
+      i1 = match[2].to_i
+      BSON::Timestamp.new(s1,i1)
+    end
+  end
+end

data/lib/mongo_oplog_backup/version.rb

@@ -1,3 +1,3 @@
 module MongoOplogBackup
-  VERSION = "0.0.12"
+  VERSION = "0.0.13"
 end

data/mongo-oplog-backup.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rspec", "~> 3.0.0"
   spec.add_development_dependency "moped", "~> 2.0"
+  spec.add_development_dependency "timecop", "~> 0.9.1"
 end

data/spec/fixtures/rotation/backup/backup.json

@@ -0,0 +1 @@
+{"backup": "backup-1504217100:18"}

data/spec/rotate_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+require 'fileutils'
+require 'timecop'
+
+describe MongoOplogBackup::Rotate do
+  SPEC_TMP='spec-tmp/backup'
+
+  let(:rotate) { MongoOplogBackup::Rotate.new(MongoOplogBackup::Config.new(dir: SPEC_TMP)) }
+  let(:backup_dir) { Pathname.new(SPEC_TMP) }
+
+  before(:each) do
+    FileUtils.mkdir_p SPEC_TMP
+    FileUtils.cp_r Dir['spec/fixtures/rotation/backup/*'], SPEC_TMP
+  end
+
+  describe '#find_backup_directories' do
+    subject { rotate.find_backup_directories }
+
+    it 'excludes directories with unexpected name format' do
+      expect(subject).to_not include(backup_dir.join('ignore-me'))
+    end
+
+    it 'excludes directories from unsuccessful backups' do
+      expect(subject).to_not include(backup_dir.join('backup-1507117250:55'))
+    end
+
+    it 'includes directories from successful backups' do
+      expect(subject).to include(backup_dir.join('backup-1498860301:15'))
+      expect(subject).to include(backup_dir.join('backup-1501538704:28'))
+      expect(subject).to include(backup_dir.join('backup-1504217100:18'))
+      expect(subject.count).to eq(3)
+    end
+  end
+
+  context 'with defaults' do
+    before do
+      Timecop.freeze(Time.utc(2017,9,1,23,0,0))
+    end
+    after do
+      Timecop.return
+    end
+
+    # Fixtures created on the 1st at 00:05:00 GMT+2
+    # 1498860301:15	2017-06-30 22:05:01 GMT
+    # 1501538704:28	2017-07-31 22:05:04 GMT
+    # 1504217100:18	2017-08-31 22:05:00 GMT
+    it 'excludes the current and previous backup set' do
+      filtered_list = rotate.filter_for_deletion(rotate.backup_list)
+
+
+      expect(filtered_list).to eq([backup_dir.join('backup-1498860301:15')])
+      # Does not include the incomplete/broken backup job: backup-1507117250:55
+      # Does not include arbitrary other directories
+    end
+
+    it 'deletes only the correct directory' do
+      rotate.perform
+
+      expect( File.exist?(File.join(SPEC_TMP, 'backup-1501538704:28' )) ).to eq(true)
+      expect( File.exist?(File.join(SPEC_TMP, 'backup-1504217100:18' )) ).to eq(true)
+      expect( File.exist?(File.join(SPEC_TMP, 'backup-1498860301:15' )) ).to eq(false)
+    end
+
+  end
+
+  context 'as a dry run' do
+    before do
+      Timecop.freeze(Time.utc(2017,9,1,23,0,0))
+    end
+    after do
+      Timecop.return
+    end
+
+    let(:config) do
+      {
+        dir: SPEC_TMP,
+        dryRun: true
+      }
+    end
+    let(:rotate) { MongoOplogBackup::Rotate.new(MongoOplogBackup::Config.new(config)) }
+
+    it 'does not delete anything.' do
+      rotate.perform
+
+      expect( File.exist?(File.join(SPEC_TMP, 'backup-1501538704:28' )) ).to eq(true)
+      expect( File.exist?(File.join(SPEC_TMP, 'backup-1504217100:18' )) ).to eq(true)
+      expect( File.exist?(File.join(SPEC_TMP, 'backup-1498860301:15' )) ).to eq(true)
+    end
+  end
+
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mongo-oplog-backup
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Ralf Kistner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-10 00:00:00.000000000 Z
+date: 2017-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bson
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: moped
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.1
 description: Periodically backup new sections of the oplog for incremental backups.
 email:
 - ralf@journeyapps.com
@@ -117,8 +131,10 @@ files:
 - lib/mongo_oplog_backup/config.rb
 - lib/mongo_oplog_backup/ext/enumerable.rb
 - lib/mongo_oplog_backup/ext/timestamp.rb
+- lib/mongo_oplog_backup/lock.rb
 - lib/mongo_oplog_backup/oplog.rb
 - lib/mongo_oplog_backup/restore.rb
+- lib/mongo_oplog_backup/rotate.rb
 - lib/mongo_oplog_backup/version.rb
 - mongo-oplog-backup.gemspec
 - oplog-last-timestamp.js
@@ -134,7 +150,13 @@ files:
 - spec/fixtures/oplog-1408088740:1-1408088810:1.bson
 - spec/fixtures/oplog-1408088810:1-1408088928:1.bson
 - spec/fixtures/oplog-merged.bson
+- spec/fixtures/rotation/backup/backup-1498860301:15/state.json
+- spec/fixtures/rotation/backup/backup-1501538704:28/state.json
+- spec/fixtures/rotation/backup/backup-1504217100:18/state.json
+- spec/fixtures/rotation/backup/backup-1507117250:55/dump/incomplete
+- spec/fixtures/rotation/backup/backup.json
 - spec/oplog_spec.rb
+- spec/rotate_spec.rb
 - spec/spec_helper.rb
 - spec/timestamp_spec.rb
 - testing.sh
@@ -158,7 +180,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Incremental backups for MongoDB using the oplog.
@@ -174,6 +196,12 @@ test_files:
 - spec/fixtures/oplog-1408088740:1-1408088810:1.bson
 - spec/fixtures/oplog-1408088810:1-1408088928:1.bson
 - spec/fixtures/oplog-merged.bson
+- spec/fixtures/rotation/backup/backup-1498860301:15/state.json
+- spec/fixtures/rotation/backup/backup-1501538704:28/state.json
+- spec/fixtures/rotation/backup/backup-1504217100:18/state.json
+- spec/fixtures/rotation/backup/backup-1507117250:55/dump/incomplete
+- spec/fixtures/rotation/backup/backup.json
 - spec/oplog_spec.rb
+- spec/rotate_spec.rb
 - spec/spec_helper.rb
 - spec/timestamp_spec.rb