mongo-oplog-backup 0.0.9 → 0.0.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/bin/mongo-oplog-backup +3 -1
- data/lib/mongo_oplog_backup/backup.rb +9 -5
- data/lib/mongo_oplog_backup/config.rb +10 -1
- data/lib/mongo_oplog_backup/oplog.rb +30 -6
- data/lib/mongo_oplog_backup/version.rb +1 -1
- data/spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz +0 -0
- data/spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz +0 -0
- data/spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz +0 -0
- data/spec/fixtures/gzip/oplog-merged-gzipped.bson +0 -0
- data/spec/oplog_spec.rb +54 -0
- metadata +10 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0c7c7a2d6fe026794174767a87b18f6ceb6e3a23
|
4
|
+
data.tar.gz: f08e6267096e438fc98c2f35e09984dfede204f9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 584ccd474c2601256d71c831623e903d80505f549787b78d4cbfa71375af35d630616ff5bad7e0b20af4729dbf2e204cbcef280cf73e72d28e62fe019c995d28
|
7
|
+
data.tar.gz: 4db03ccb06d66a6295fde3c79a75066a4ded1eb3830cd0416e46b9f7a42c99897a88a0d71035100e61960d1629c0e8291447be0f4e42157413bcdaf1da7bb78a
|
data/README.md
CHANGED
@@ -14,7 +14,7 @@ a single file that can be stored on your preferred medium, for example Amazon S3
|
|
14
14
|
or an FTP site. This project only provides the tools to produce the backup files,
|
15
15
|
and it's up to you to transfer it to a backup medium.
|
16
16
|
|
17
|
-
|
17
|
+
Internally the `mongodump` command is used for the backup operations. Initially
|
18
18
|
a full dump is performed, after which incremetal backups are performed by backing
|
19
19
|
up new sections of the oplog. Only the standard BSON format from mongodump is used.
|
20
20
|
|
data/bin/mongo-oplog-backup
CHANGED
@@ -14,6 +14,7 @@ opts = Slop.parse(help: true, strict: true) do
|
|
14
14
|
on :oplog, 'Force oplog backup'
|
15
15
|
|
16
16
|
on :f, :file, 'Configuration file for common defaults', argument: :required
|
17
|
+
on :gzip, "Use gzip compression"
|
17
18
|
on :ssl, "Connect to a mongod instance over an SSL connection"
|
18
19
|
on :sslAllowInvalidCertificates, "Allow connections to a mongod instance with an invalid certificate"
|
19
20
|
on :sslCAFile, "Specifies a Certificate Authority file for validating the SSL certificate provided by the mongod instance.", argument: :required
|
@@ -26,6 +27,7 @@ opts = Slop.parse(help: true, strict: true) do
|
|
26
27
|
dir = opts[:dir] || 'backup'
|
27
28
|
config_opts = {
|
28
29
|
dir: dir,
|
30
|
+
gzip: opts.gzip?,
|
29
31
|
ssl: opts.ssl?,
|
30
32
|
sslAllowInvalidCertificates: opts.sslAllowInvalidCertificates?
|
31
33
|
}
|
@@ -61,7 +63,7 @@ opts = Slop.parse(help: true, strict: true) do
|
|
61
63
|
MongoOplogBackup::Oplog.merge_backup(dir)
|
62
64
|
puts
|
63
65
|
puts "Restore the backup with: "
|
64
|
-
puts "mongorestore [--drop] --oplogReplay #{File.join(dir, 'dump')}"
|
66
|
+
puts "mongorestore [--drop] [--gzip] --oplogReplay #{File.join(dir, 'dump')}"
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
@@ -53,9 +53,10 @@ module MongoOplogBackup
|
|
53
53
|
|
54
54
|
query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]
|
55
55
|
|
56
|
-
|
57
|
-
|
58
|
-
|
56
|
+
dump_args = ['--out', config.oplog_dump_folder, '--db', 'local', '--collection', 'oplog.rs']
|
57
|
+
dump_args += query
|
58
|
+
dump_args << '--gzip' if config.use_compression?
|
59
|
+
config.mongodump(dump_args)
|
59
60
|
|
60
61
|
unless File.exists? config.oplog_dump
|
61
62
|
raise "mongodump failed"
|
@@ -89,6 +90,7 @@ module MongoOplogBackup
|
|
89
90
|
result[:empty] = true
|
90
91
|
else
|
91
92
|
outfile = "oplog-#{first}-#{last}.bson"
|
93
|
+
outfile += '.gz' if config.use_compression?
|
92
94
|
full_path = File.join(backup_folder, outfile)
|
93
95
|
FileUtils.mkdir_p backup_folder
|
94
96
|
FileUtils.mv config.oplog_dump, full_path
|
@@ -124,7 +126,9 @@ module MongoOplogBackup
|
|
124
126
|
raise "Backup folder '#{backup_folder}' already exists; not performing backup."
|
125
127
|
end
|
126
128
|
dump_folder = File.join(backup_folder, 'dump')
|
127
|
-
|
129
|
+
dump_args = ['--out', dump_folder]
|
130
|
+
dump_args << '--gzip' if config.use_compression?
|
131
|
+
result = config.mongodump(dump_args)
|
128
132
|
unless File.directory? dump_folder
|
129
133
|
MongoOplogBackup.log.error 'Backup folder does not exist'
|
130
134
|
raise 'Full backup failed'
|
@@ -159,7 +163,7 @@ module MongoOplogBackup
|
|
159
163
|
end
|
160
164
|
|
161
165
|
if mode == :oplog
|
162
|
-
raise "Unknown backup position - cannot perform oplog backup." unless have_backup
|
166
|
+
raise "Unknown backup position - cannot perform oplog backup. Have you completed a full backup?" unless have_backup
|
163
167
|
MongoOplogBackup.log.info "Performing incremental oplog backup"
|
164
168
|
lock(File.join(backup_folder, 'backup.lock')) do
|
165
169
|
result = backup_oplog
|
@@ -14,6 +14,7 @@ module MongoOplogBackup
|
|
14
14
|
options = {}
|
15
15
|
unless file.nil?
|
16
16
|
conf = YAML.load_file(file)
|
17
|
+
options[:gzip] = conf["gzip"] unless conf["gzip"].nil?
|
17
18
|
options[:ssl] = conf["ssl"] unless conf["ssl"].nil?
|
18
19
|
options[:sslAllowInvalidCertificates] = conf["sslAllowInvalidCertificates"] unless conf["sslAllowInvalidCertificates"].nil?
|
19
20
|
options[:sslCAFile] = conf["sslCAFile"] unless conf["sslCAFile"].nil?
|
@@ -30,6 +31,10 @@ module MongoOplogBackup
|
|
30
31
|
options[:dir]
|
31
32
|
end
|
32
33
|
|
34
|
+
def use_compression?
|
35
|
+
!!options[:gzip]
|
36
|
+
end
|
37
|
+
|
33
38
|
def command_line_options
|
34
39
|
args = []
|
35
40
|
args << '--ssl' if options[:ssl]
|
@@ -48,7 +53,11 @@ module MongoOplogBackup
|
|
48
53
|
end
|
49
54
|
|
50
55
|
def oplog_dump
|
51
|
-
|
56
|
+
if use_compression?
|
57
|
+
File.join(oplog_dump_folder, 'local/oplog.rs.bson.gz')
|
58
|
+
else
|
59
|
+
File.join(oplog_dump_folder, 'local/oplog.rs.bson')
|
60
|
+
end
|
52
61
|
end
|
53
62
|
|
54
63
|
def global_state_file
|
@@ -1,11 +1,19 @@
|
|
1
|
+
require 'zlib'
|
1
2
|
module MongoOplogBackup
|
2
3
|
module Oplog
|
3
4
|
def self.each_document(filename)
|
4
|
-
|
5
|
+
yield_bson_document = Proc.new do |stream|
|
5
6
|
while !stream.eof?
|
7
|
+
# FIXME: Since bson4, from_bson takes a ByteArray instead of a StringIO
|
6
8
|
yield BSON::Document.from_bson(stream)
|
7
9
|
end
|
8
10
|
end
|
11
|
+
|
12
|
+
if gzip_fingerprint(filename)
|
13
|
+
Zlib::GzipReader.open(filename, &yield_bson_document)
|
14
|
+
else
|
15
|
+
File.open(filename, 'rb', &yield_bson_document)
|
16
|
+
end
|
9
17
|
end
|
10
18
|
|
11
19
|
def self.oplog_timestamps(filename)
|
@@ -18,7 +26,7 @@ module MongoOplogBackup
|
|
18
26
|
timestamps
|
19
27
|
end
|
20
28
|
|
21
|
-
FILENAME_RE = /\/oplog-(\d+):(\d+)-(\d+):(\d+)\.bson
|
29
|
+
FILENAME_RE = /\/oplog-(\d+):(\d+)-(\d+):(\d+)\.bson(?:\.gz)?\z/
|
22
30
|
|
23
31
|
def self.timestamps_from_filename filename
|
24
32
|
match = FILENAME_RE.match(filename)
|
@@ -38,8 +46,9 @@ module MongoOplogBackup
|
|
38
46
|
def self.merge(target, source_files, options={})
|
39
47
|
limit = options[:limit] # TODO: use
|
40
48
|
force = options[:force]
|
49
|
+
compress = !!options[:gzip]
|
41
50
|
|
42
|
-
|
51
|
+
process_output = Proc.new do |output|
|
43
52
|
last_timestamp = nil
|
44
53
|
first = true
|
45
54
|
|
@@ -65,6 +74,10 @@ module MongoOplogBackup
|
|
65
74
|
Oplog.each_document(filename) do |doc|
|
66
75
|
timestamp = doc['ts']
|
67
76
|
first_file_timestamp = timestamp if first_file_timestamp.nil?
|
77
|
+
|
78
|
+
# gzip stores the mtime in the header, so we set it explicity for consistency between runs.
|
79
|
+
output.mtime = first_file_timestamp.seconds if output.mtime.to_i == 0
|
80
|
+
|
68
81
|
if !last_timestamp.nil? && timestamp <= last_timestamp
|
69
82
|
skipped += 1
|
70
83
|
elsif !last_file_timestamp.nil? && timestamp <= last_file_timestamp
|
@@ -90,10 +103,15 @@ module MongoOplogBackup
|
|
90
103
|
first = false
|
91
104
|
end
|
92
105
|
end
|
106
|
+
if (compress)
|
107
|
+
Zlib::GzipWriter.open(target, &process_output)
|
108
|
+
else
|
109
|
+
File.open(target, 'wb', &process_output)
|
110
|
+
end
|
93
111
|
end
|
94
112
|
|
95
113
|
def self.find_oplogs(dir)
|
96
|
-
files = Dir.glob(File.join(dir, 'oplog-*.bson'))
|
114
|
+
files = Dir.glob(File.join(dir, 'oplog-*.bson*'))
|
97
115
|
files.keep_if {|name| name =~ FILENAME_RE}
|
98
116
|
files.sort! {|a, b| timestamps_from_filename(a)[:first] <=> timestamps_from_filename(b)[:first]}
|
99
117
|
files
|
@@ -101,9 +119,15 @@ module MongoOplogBackup
|
|
101
119
|
|
102
120
|
def self.merge_backup(dir)
|
103
121
|
oplogs = find_oplogs(dir)
|
104
|
-
|
122
|
+
compress_target = oplogs.any? { |o| o.end_with?('.gz') }
|
123
|
+
target = File.join(dir, 'dump', 'oplog.bson') # Mongorestore expects this filename, without a gzip suffix.
|
105
124
|
FileUtils.mkdir_p(File.join(dir, 'dump'))
|
106
|
-
merge(target, oplogs)
|
125
|
+
merge(target, oplogs, {gzip: compress_target})
|
126
|
+
end
|
127
|
+
|
128
|
+
def self.gzip_fingerprint filename
|
129
|
+
bytes = File.read(filename, 2, 0)
|
130
|
+
bytes[0] == "\x1f".force_encoding('BINARY') && bytes[1] == "\x8b".force_encoding('BINARY')
|
107
131
|
end
|
108
132
|
|
109
133
|
end
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/spec/oplog_spec.rb
CHANGED
@@ -55,4 +55,58 @@ describe MongoOplogBackup::Oplog do
|
|
55
55
|
|
56
56
|
'spec-tmp/backup/dump/oplog.bson'.should be_same_oplog_as oplog_merged
|
57
57
|
end
|
58
|
+
|
59
|
+
|
60
|
+
|
61
|
+
context 'with gzipped oplogs' do
|
62
|
+
let(:oplog1) { 'spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz'}
|
63
|
+
let(:oplog2) { 'spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz'}
|
64
|
+
let(:oplog3) { 'spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz'}
|
65
|
+
let(:oplog_merged) { 'spec/fixtures/gzip/oplog-merged-gzipped.bson'}
|
66
|
+
|
67
|
+
it 'should extract oplog timestamps' do
|
68
|
+
timestamps = MongoOplogBackup::Oplog.oplog_timestamps(oplog1)
|
69
|
+
timestamps.should == [
|
70
|
+
BSON::Timestamp.new(1479827504, 7),
|
71
|
+
BSON::Timestamp.new(1479827515, 1),
|
72
|
+
BSON::Timestamp.new(1479827517, 1),
|
73
|
+
BSON::Timestamp.new(1479827518, 1)
|
74
|
+
]
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'should merge oplogs' do
|
78
|
+
merged_out = 'spec-tmp/oplog-merged-gzipped.bson'
|
79
|
+
MongoOplogBackup::Oplog.merge(merged_out, [oplog1, oplog2, oplog3], {gzip: true})
|
80
|
+
|
81
|
+
expected_timestamps =
|
82
|
+
MongoOplogBackup::Oplog.oplog_timestamps(oplog1) +
|
83
|
+
MongoOplogBackup::Oplog.oplog_timestamps(oplog2) +
|
84
|
+
MongoOplogBackup::Oplog.oplog_timestamps(oplog3)
|
85
|
+
|
86
|
+
expected_timestamps.uniq!
|
87
|
+
expected_timestamps.sort! # Not sure if uniq! modifies the order
|
88
|
+
|
89
|
+
actual_timestamps = MongoOplogBackup::Oplog.oplog_timestamps(merged_out)
|
90
|
+
actual_timestamps.should == expected_timestamps
|
91
|
+
|
92
|
+
merged_out.should be_same_oplog_as oplog_merged
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'should parse timestamps from a filename' do
|
96
|
+
timestamps = MongoOplogBackup::Oplog.timestamps_from_filename('some/oplog-1408088734:1-1408088740:52.bson.gz')
|
97
|
+
timestamps.should == {
|
98
|
+
first: BSON::Timestamp.new(1408088734, 1),
|
99
|
+
last: BSON::Timestamp.new(1408088740, 52)
|
100
|
+
}
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should merge a backup folder" do
|
104
|
+
FileUtils.mkdir_p 'spec-tmp/backup-zipped'
|
105
|
+
FileUtils.cp_r Dir['spec/fixtures/gzip/oplog-*.bson.gz'], 'spec-tmp/backup-zipped/'
|
106
|
+
|
107
|
+
MongoOplogBackup::Oplog.merge_backup('spec-tmp/backup-zipped')
|
108
|
+
|
109
|
+
'spec-tmp/backup-zipped/dump/oplog.bson'.should be_same_oplog_as oplog_merged
|
110
|
+
end
|
111
|
+
end
|
58
112
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongo-oplog-backup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ralf Kistner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bson
|
@@ -125,6 +125,10 @@ files:
|
|
125
125
|
- spec/backup_spec.rb
|
126
126
|
- spec/command_spec.rb
|
127
127
|
- spec/enumerable_spec.rb
|
128
|
+
- spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz
|
129
|
+
- spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz
|
130
|
+
- spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz
|
131
|
+
- spec/fixtures/gzip/oplog-merged-gzipped.bson
|
128
132
|
- spec/fixtures/oplog-1408088734:1-1408088740:1.bson
|
129
133
|
- spec/fixtures/oplog-1408088740:1-1408088810:1.bson
|
130
134
|
- spec/fixtures/oplog-1408088810:1-1408088928:1.bson
|
@@ -160,6 +164,10 @@ test_files:
|
|
160
164
|
- spec/backup_spec.rb
|
161
165
|
- spec/command_spec.rb
|
162
166
|
- spec/enumerable_spec.rb
|
167
|
+
- spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz
|
168
|
+
- spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz
|
169
|
+
- spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz
|
170
|
+
- spec/fixtures/gzip/oplog-merged-gzipped.bson
|
163
171
|
- spec/fixtures/oplog-1408088734:1-1408088740:1.bson
|
164
172
|
- spec/fixtures/oplog-1408088740:1-1408088810:1.bson
|
165
173
|
- spec/fixtures/oplog-1408088810:1-1408088928:1.bson
|