mongo-oplog-backup 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +13 -0
- data/bin/mongo-oplog-backup +20 -0
- data/lib/mongo_oplog_backup.rb +2 -0
- data/lib/mongo_oplog_backup/backup.rb +1 -14
- data/lib/mongo_oplog_backup/lock.rb +21 -0
- data/lib/mongo_oplog_backup/rotate.rb +90 -0
- data/lib/mongo_oplog_backup/version.rb +1 -1
- data/mongo-oplog-backup.gemspec +2 -1
- data/spec/fixtures/rotation/backup/backup-1498860301:15/state.json +0 -0
- data/spec/fixtures/rotation/backup/backup-1501538704:28/state.json +0 -0
- data/spec/fixtures/rotation/backup/backup-1504217100:18/state.json +0 -0
- data/spec/fixtures/rotation/backup/backup-1507117250:55/dump/incomplete +0 -0
- data/spec/fixtures/rotation/backup/backup.json +1 -0
- data/spec/rotate_spec.rb +92 -0
- metadata +33 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 614d931c8a55e33ca69b78eb20c38204c2668d50
|
4
|
+
data.tar.gz: 5807c75c2173be443b36ceee09ca5e38c545a4c6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dd246853e1c69ceac5a503d6bf9a684c9bc6ab44debdffaa0ced2389521194cdaf9751b9c72f61dd96b912fe2f3857a19c9c4e99b979975f51443ace77e4fd74
|
7
|
+
data.tar.gz: 7713c2c5de857b47eaaddd6b8304bad7d7c97f56aa176ba71abfddcba9159de003f1b5c72718de2ff825c350aaea15bed215a6401dfa3de683afcc2be135e381
|
data/README.md
CHANGED
@@ -66,6 +66,19 @@ That said, there have been bugs in the past that caused the oplog to not be idem
|
|
66
66
|
in some edge cases. Therefore it is recommended to stop the secondary before performing
|
67
67
|
a full backup.
|
68
68
|
|
69
|
+
## Rotation
|
70
|
+
|
71
|
+
Older backups can be automatically deleted with `mongo-oplog-backup rotate`.
|
72
|
+
|
73
|
+
For example, given the full backup schedule above and a recovery point objective of 4 weeks, any full backup sets
|
74
|
+
older than 4 weeks (excluding the currently active backup set and the previous backup set) can be deleted.
|
75
|
+
|
76
|
+
A sample cron script to clean-up any older backups about an hour before the weekly full backup:
|
77
|
+
|
78
|
+
0 23 * * 0 /path/to/ruby/bin/mongo-oplog-backup rotate --dir /path/to/backup/location --keep_days=28 >> /path/to/backup.log
|
79
|
+
|
80
|
+
It is strongly recommended to use the `dryRun` option to thoroughly test your particular configuration and schedule.
|
81
|
+
|
69
82
|
## To restore
|
70
83
|
|
71
84
|
mongo-oplog-backup merge --dir mybackup/backup-<timestamp>
|
data/bin/mongo-oplog-backup
CHANGED
@@ -73,6 +73,26 @@ opts = Slop.parse(help: true, strict: true) do
|
|
73
73
|
end
|
74
74
|
end
|
75
75
|
|
76
|
+
command 'rotate' do
|
77
|
+
banner "Delete old backup sets.\nUsage: mongo-oplog-backup rotate [options]"
|
78
|
+
on :v, :verbose, "Enable verbose mode"
|
79
|
+
on :d, :dir, "Directory where backup files are stored. Defaults to 'backup'.", argument: :required
|
80
|
+
on :keepDays, "Minimum number of days of backups to keep (recovery point objective). This should be AT LEAST equal to the Full backup schedule.", default: '32', argument: :required
|
81
|
+
on :dryRun, "Perform a trial run with no changes made."
|
82
|
+
|
83
|
+
run do |opts, args|
|
84
|
+
dir = opts[:dir] || 'backup'
|
85
|
+
config_opts = {
|
86
|
+
dir: dir,
|
87
|
+
keepDays: (opts[:keepDays].to_i rescue nil),
|
88
|
+
dryRun: opts.dryRun?
|
89
|
+
}
|
90
|
+
config = MongoOplogBackup::Config.new(config_opts)
|
91
|
+
rotate = MongoOplogBackup::Rotate.new(config)
|
92
|
+
rotate.perform
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
76
96
|
command 'restore' do
|
77
97
|
banner 'Usage: mongo-oplog-backup restore [options]'
|
78
98
|
on :v, :verbose, 'Enable verbose mode'
|
data/lib/mongo_oplog_backup.rb
CHANGED
@@ -4,11 +4,13 @@ require 'mongo_oplog_backup/version'
|
|
4
4
|
require 'mongo_oplog_backup/ext/enumerable'
|
5
5
|
require 'mongo_oplog_backup/ext/timestamp'
|
6
6
|
|
7
|
+
require 'mongo_oplog_backup/lock'
|
7
8
|
require 'mongo_oplog_backup/command'
|
8
9
|
require 'mongo_oplog_backup/config'
|
9
10
|
require 'mongo_oplog_backup/backup'
|
10
11
|
require 'mongo_oplog_backup/oplog'
|
11
12
|
require 'mongo_oplog_backup/restore'
|
13
|
+
require 'mongo_oplog_backup/rotate'
|
12
14
|
|
13
15
|
module MongoOplogBackup
|
14
16
|
def self.log
|
@@ -2,11 +2,9 @@ require 'json'
|
|
2
2
|
require 'fileutils'
|
3
3
|
require 'mongo_oplog_backup/oplog'
|
4
4
|
|
5
|
-
class LockError < StandardError
|
6
|
-
end
|
7
|
-
|
8
5
|
module MongoOplogBackup
|
9
6
|
class Backup
|
7
|
+
include Lockable
|
10
8
|
attr_reader :config, :backup_name
|
11
9
|
|
12
10
|
def backup_folder
|
@@ -33,17 +31,6 @@ module MongoOplogBackup
|
|
33
31
|
File.write(state_file, state.to_json)
|
34
32
|
end
|
35
33
|
|
36
|
-
def lock(lockname, &block)
|
37
|
-
File.open(lockname, File::RDWR|File::CREAT, 0644) do |file|
|
38
|
-
# Get a non-blocking lock
|
39
|
-
got_lock = file.flock(File::LOCK_EX|File::LOCK_NB)
|
40
|
-
if got_lock == false
|
41
|
-
raise LockError, "Failed to acquire lock - another backup may be busy"
|
42
|
-
end
|
43
|
-
yield
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
34
|
def backup_oplog(options={})
|
48
35
|
raise ArgumentError, "No state in #{backup_name}" unless File.exists? state_file
|
49
36
|
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module MongoOplogBackup
|
2
|
+
class LockError < StandardError
|
3
|
+
end
|
4
|
+
|
5
|
+
module Lockable
|
6
|
+
def self.included(base)
|
7
|
+
base.extend(self)
|
8
|
+
end
|
9
|
+
|
10
|
+
def lock(lockname, &block)
|
11
|
+
File.open(lockname, File::RDWR|File::CREAT, 0644) do |file|
|
12
|
+
# Get a non-blocking lock
|
13
|
+
got_lock = file.flock(File::LOCK_EX|File::LOCK_NB)
|
14
|
+
if got_lock == false
|
15
|
+
raise LockError, "Failed to acquire lock - another backup may be busy"
|
16
|
+
end
|
17
|
+
yield
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
|
3
|
+
module MongoOplogBackup
|
4
|
+
class Rotate
|
5
|
+
include Lockable
|
6
|
+
attr_reader :config, :backup_list
|
7
|
+
DAY = 86400
|
8
|
+
RECOVERY_POINT_OBJECTIVE = 32 * DAY # Longest month + 1
|
9
|
+
KEEP_MINIMUM_SETS = 2 # Current & Previous
|
10
|
+
|
11
|
+
BACKUP_DIR_NAME_FORMAT = /\Abackup-\d+:\d+\z/
|
12
|
+
|
13
|
+
def initialize(config)
|
14
|
+
@config = config
|
15
|
+
@dry_run = !!@config.options[:dryRun]
|
16
|
+
@backup_list = find_backup_directories
|
17
|
+
if @config.options[:keepDays].nil?
|
18
|
+
@recovery_point_objective = RECOVERY_POINT_OBJECTIVE
|
19
|
+
else
|
20
|
+
@recovery_point_objective = @config.options[:keepDays] * DAY
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def current_backup_name
|
25
|
+
if @current_backup_name.nil?
|
26
|
+
state_file = config.global_state_file
|
27
|
+
state = JSON.parse(File.read(state_file)) rescue nil
|
28
|
+
state ||= {}
|
29
|
+
@current_backup_name = state['backup']
|
30
|
+
end
|
31
|
+
@current_backup_name
|
32
|
+
end
|
33
|
+
|
34
|
+
def perform
|
35
|
+
lock(config.global_lock_file) do
|
36
|
+
MongoOplogBackup.log.info "Rotating out old backups."
|
37
|
+
|
38
|
+
if @backup_list.count >= 0 && @backup_list.count <= KEEP_MINIMUM_SETS
|
39
|
+
MongoOplogBackup.log.info "Too few backup sets to automatically rotate."
|
40
|
+
elsif @backup_list.count > KEEP_MINIMUM_SETS
|
41
|
+
filter_for_deletion(@backup_list).each do |path|
|
42
|
+
MongoOplogBackup.log.info "#{@dry_run ? '[DRYRUN] Would delete' : 'Deleting'} #{path}."
|
43
|
+
begin
|
44
|
+
FileUtils.remove_entry_secure(path) unless @dry_run
|
45
|
+
rescue StandardError => e
|
46
|
+
MongoOplogBackup.log.error "Delete failed: #{e.message}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
MongoOplogBackup.log.info "Rotating out old backups completed."
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# Lists subdirectories in the backup location that match the backup set naming format and appear to have completed
|
56
|
+
# successfully.
|
57
|
+
# @return [Array<Pathname>] backup directories.
|
58
|
+
def find_backup_directories
|
59
|
+
dirs = Pathname.new(@config.backup_dir).children.select(&:directory?)
|
60
|
+
dirs = dirs.select { |dir| dir.basename.to_s =~ BACKUP_DIR_NAME_FORMAT }
|
61
|
+
dirs = dirs.select { |dir| File.exist?(File.join(dir, 'state.json')) }
|
62
|
+
dirs.sort
|
63
|
+
end
|
64
|
+
|
65
|
+
# @param [Array<Pathname>] source_list List of Pathnames for the full backup sets in the backup directory.
|
66
|
+
# @return [Array<Pathname]
|
67
|
+
def filter_for_deletion(source_list)
|
68
|
+
source_list = source_list.sort.reverse.drop(KEEP_MINIMUM_SETS) # Keep a minimum number of full backups
|
69
|
+
# The most recent dir might not be the active one (eg. if the mongodump fails). Ensure that it is excluded.
|
70
|
+
source_list = source_list.reject { |path| path.basename.to_s == current_backup_name }
|
71
|
+
|
72
|
+
source_list.select {|path| age_of_backup_in_seconds(path.basename) > @recovery_point_objective }
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
|
77
|
+
def age_of_backup_in_seconds(path)
|
78
|
+
Time.now.to_i - timestamp_from_string(path.to_s).seconds
|
79
|
+
end
|
80
|
+
|
81
|
+
# Accepts: <seconds>[:ordinal]
|
82
|
+
def timestamp_from_string(string)
|
83
|
+
match = /(\d+)(?::(\d+))?/.match(string)
|
84
|
+
return nil unless match
|
85
|
+
s1 = match[1].to_i
|
86
|
+
i1 = match[2].to_i
|
87
|
+
BSON::Timestamp.new(s1,i1)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
data/mongo-oplog-backup.gemspec
CHANGED
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
|
|
23
23
|
|
24
24
|
spec.add_development_dependency "bundler", "~> 1.5"
|
25
25
|
spec.add_development_dependency "rake"
|
26
|
-
spec.add_development_dependency "rspec", "~> 3.0"
|
26
|
+
spec.add_development_dependency "rspec", "~> 3.0.0"
|
27
27
|
spec.add_development_dependency "moped", "~> 2.0"
|
28
|
+
spec.add_development_dependency "timecop", "~> 0.9.1"
|
28
29
|
end
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
@@ -0,0 +1 @@
|
|
1
|
+
{"backup": "backup-1504217100:18"}
|
data/spec/rotate_spec.rb
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'timecop'
|
4
|
+
|
5
|
+
describe MongoOplogBackup::Rotate do
|
6
|
+
SPEC_TMP='spec-tmp/backup'
|
7
|
+
|
8
|
+
let(:rotate) { MongoOplogBackup::Rotate.new(MongoOplogBackup::Config.new(dir: SPEC_TMP)) }
|
9
|
+
let(:backup_dir) { Pathname.new(SPEC_TMP) }
|
10
|
+
|
11
|
+
before(:each) do
|
12
|
+
FileUtils.mkdir_p SPEC_TMP
|
13
|
+
FileUtils.cp_r Dir['spec/fixtures/rotation/backup/*'], SPEC_TMP
|
14
|
+
end
|
15
|
+
|
16
|
+
describe '#find_backup_directories' do
|
17
|
+
subject { rotate.find_backup_directories }
|
18
|
+
|
19
|
+
it 'excludes directories with unexpected name format' do
|
20
|
+
expect(subject).to_not include(backup_dir.join('ignore-me'))
|
21
|
+
end
|
22
|
+
|
23
|
+
it 'excludes directories from unsuccessful backups' do
|
24
|
+
expect(subject).to_not include(backup_dir.join('backup-1507117250:55'))
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'includes directories from successful backups' do
|
28
|
+
expect(subject).to include(backup_dir.join('backup-1498860301:15'))
|
29
|
+
expect(subject).to include(backup_dir.join('backup-1501538704:28'))
|
30
|
+
expect(subject).to include(backup_dir.join('backup-1504217100:18'))
|
31
|
+
expect(subject.count).to eq(3)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
context 'with defaults' do
|
36
|
+
before do
|
37
|
+
Timecop.freeze(Time.utc(2017,9,1,23,0,0))
|
38
|
+
end
|
39
|
+
after do
|
40
|
+
Timecop.return
|
41
|
+
end
|
42
|
+
|
43
|
+
# Fixtures created on the 1st at 00:05:00 GMT+2
|
44
|
+
# 1498860301:15 2017-06-30 22:05:01 GMT
|
45
|
+
# 1501538704:28 2017-07-31 22:05:04 GMT
|
46
|
+
# 1504217100:18 2017-08-31 22:05:00 GMT
|
47
|
+
it 'excludes the current and previous backup set' do
|
48
|
+
filtered_list = rotate.filter_for_deletion(rotate.backup_list)
|
49
|
+
|
50
|
+
|
51
|
+
expect(filtered_list).to eq([backup_dir.join('backup-1498860301:15')])
|
52
|
+
# Does not include the incomplete/broken backup job: backup-1507117250:55
|
53
|
+
# Does not include arbitrary other directories
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'deletes only the correct directory' do
|
57
|
+
rotate.perform
|
58
|
+
|
59
|
+
expect( File.exist?(File.join(SPEC_TMP, 'backup-1501538704:28' )) ).to eq(true)
|
60
|
+
expect( File.exist?(File.join(SPEC_TMP, 'backup-1504217100:18' )) ).to eq(true)
|
61
|
+
expect( File.exist?(File.join(SPEC_TMP, 'backup-1498860301:15' )) ).to eq(false)
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'as a dry run' do
|
67
|
+
before do
|
68
|
+
Timecop.freeze(Time.utc(2017,9,1,23,0,0))
|
69
|
+
end
|
70
|
+
after do
|
71
|
+
Timecop.return
|
72
|
+
end
|
73
|
+
|
74
|
+
let(:config) do
|
75
|
+
{
|
76
|
+
dir: SPEC_TMP,
|
77
|
+
dryRun: true
|
78
|
+
}
|
79
|
+
end
|
80
|
+
let(:rotate) { MongoOplogBackup::Rotate.new(MongoOplogBackup::Config.new(config)) }
|
81
|
+
|
82
|
+
it 'does not delete anything.' do
|
83
|
+
rotate.perform
|
84
|
+
|
85
|
+
expect( File.exist?(File.join(SPEC_TMP, 'backup-1501538704:28' )) ).to eq(true)
|
86
|
+
expect( File.exist?(File.join(SPEC_TMP, 'backup-1504217100:18' )) ).to eq(true)
|
87
|
+
expect( File.exist?(File.join(SPEC_TMP, 'backup-1498860301:15' )) ).to eq(true)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
|
92
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongo-oplog-backup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ralf Kistner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-10-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bson
|
@@ -72,14 +72,14 @@ dependencies:
|
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
75
|
+
version: 3.0.0
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
82
|
+
version: 3.0.0
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: moped
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '2.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: timecop
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 0.9.1
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 0.9.1
|
97
111
|
description: Periodically backup new sections of the oplog for incremental backups.
|
98
112
|
email:
|
99
113
|
- ralf@journeyapps.com
|
@@ -117,8 +131,10 @@ files:
|
|
117
131
|
- lib/mongo_oplog_backup/config.rb
|
118
132
|
- lib/mongo_oplog_backup/ext/enumerable.rb
|
119
133
|
- lib/mongo_oplog_backup/ext/timestamp.rb
|
134
|
+
- lib/mongo_oplog_backup/lock.rb
|
120
135
|
- lib/mongo_oplog_backup/oplog.rb
|
121
136
|
- lib/mongo_oplog_backup/restore.rb
|
137
|
+
- lib/mongo_oplog_backup/rotate.rb
|
122
138
|
- lib/mongo_oplog_backup/version.rb
|
123
139
|
- mongo-oplog-backup.gemspec
|
124
140
|
- oplog-last-timestamp.js
|
@@ -134,7 +150,13 @@ files:
|
|
134
150
|
- spec/fixtures/oplog-1408088740:1-1408088810:1.bson
|
135
151
|
- spec/fixtures/oplog-1408088810:1-1408088928:1.bson
|
136
152
|
- spec/fixtures/oplog-merged.bson
|
153
|
+
- spec/fixtures/rotation/backup/backup-1498860301:15/state.json
|
154
|
+
- spec/fixtures/rotation/backup/backup-1501538704:28/state.json
|
155
|
+
- spec/fixtures/rotation/backup/backup-1504217100:18/state.json
|
156
|
+
- spec/fixtures/rotation/backup/backup-1507117250:55/dump/incomplete
|
157
|
+
- spec/fixtures/rotation/backup/backup.json
|
137
158
|
- spec/oplog_spec.rb
|
159
|
+
- spec/rotate_spec.rb
|
138
160
|
- spec/spec_helper.rb
|
139
161
|
- spec/timestamp_spec.rb
|
140
162
|
- testing.sh
|
@@ -158,7 +180,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
158
180
|
version: '0'
|
159
181
|
requirements: []
|
160
182
|
rubyforge_project:
|
161
|
-
rubygems_version: 2.
|
183
|
+
rubygems_version: 2.6.13
|
162
184
|
signing_key:
|
163
185
|
specification_version: 4
|
164
186
|
summary: Incremental backups for MongoDB using the oplog.
|
@@ -174,6 +196,12 @@ test_files:
|
|
174
196
|
- spec/fixtures/oplog-1408088740:1-1408088810:1.bson
|
175
197
|
- spec/fixtures/oplog-1408088810:1-1408088928:1.bson
|
176
198
|
- spec/fixtures/oplog-merged.bson
|
199
|
+
- spec/fixtures/rotation/backup/backup-1498860301:15/state.json
|
200
|
+
- spec/fixtures/rotation/backup/backup-1501538704:28/state.json
|
201
|
+
- spec/fixtures/rotation/backup/backup-1504217100:18/state.json
|
202
|
+
- spec/fixtures/rotation/backup/backup-1507117250:55/dump/incomplete
|
203
|
+
- spec/fixtures/rotation/backup/backup.json
|
177
204
|
- spec/oplog_spec.rb
|
205
|
+
- spec/rotate_spec.rb
|
178
206
|
- spec/spec_helper.rb
|
179
207
|
- spec/timestamp_spec.rb
|