mongo-oplog-backup 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/bin/mongo-oplog-backup +3 -1
- data/lib/mongo_oplog_backup/backup.rb +9 -5
- data/lib/mongo_oplog_backup/config.rb +10 -1
- data/lib/mongo_oplog_backup/oplog.rb +30 -6
- data/lib/mongo_oplog_backup/version.rb +1 -1
- data/spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz +0 -0
- data/spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz +0 -0
- data/spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz +0 -0
- data/spec/fixtures/gzip/oplog-merged-gzipped.bson +0 -0
- data/spec/oplog_spec.rb +54 -0
- metadata +10 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0c7c7a2d6fe026794174767a87b18f6ceb6e3a23
|
4
|
+
data.tar.gz: f08e6267096e438fc98c2f35e09984dfede204f9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 584ccd474c2601256d71c831623e903d80505f549787b78d4cbfa71375af35d630616ff5bad7e0b20af4729dbf2e204cbcef280cf73e72d28e62fe019c995d28
|
7
|
+
data.tar.gz: 4db03ccb06d66a6295fde3c79a75066a4ded1eb3830cd0416e46b9f7a42c99897a88a0d71035100e61960d1629c0e8291447be0f4e42157413bcdaf1da7bb78a
|
data/README.md
CHANGED
@@ -14,7 +14,7 @@ a single file that can be stored on your preferred medium, for example Amazon S3
|
|
14
14
|
or an FTP site. This project only provides the tools to produce the backup files,
|
15
15
|
and it's up to you to transfer it to a backup medium.
|
16
16
|
|
17
|
-
|
17
|
+
Internally the `mongodump` command is used for the backup operations. Initially
|
18
18
|
a full dump is performed, after which incremetal backups are performed by backing
|
19
19
|
up new sections of the oplog. Only the standard BSON format from mongodump is used.
|
20
20
|
|
data/bin/mongo-oplog-backup
CHANGED
@@ -14,6 +14,7 @@ opts = Slop.parse(help: true, strict: true) do
|
|
14
14
|
on :oplog, 'Force oplog backup'
|
15
15
|
|
16
16
|
on :f, :file, 'Configuration file for common defaults', argument: :required
|
17
|
+
on :gzip, "Use gzip compression"
|
17
18
|
on :ssl, "Connect to a mongod instance over an SSL connection"
|
18
19
|
on :sslAllowInvalidCertificates, "Allow connections to a mongod instance with an invalid certificate"
|
19
20
|
on :sslCAFile, "Specifies a Certificate Authority file for validating the SSL certificate provided by the mongod instance.", argument: :required
|
@@ -26,6 +27,7 @@ opts = Slop.parse(help: true, strict: true) do
|
|
26
27
|
dir = opts[:dir] || 'backup'
|
27
28
|
config_opts = {
|
28
29
|
dir: dir,
|
30
|
+
gzip: opts.gzip?,
|
29
31
|
ssl: opts.ssl?,
|
30
32
|
sslAllowInvalidCertificates: opts.sslAllowInvalidCertificates?
|
31
33
|
}
|
@@ -61,7 +63,7 @@ opts = Slop.parse(help: true, strict: true) do
|
|
61
63
|
MongoOplogBackup::Oplog.merge_backup(dir)
|
62
64
|
puts
|
63
65
|
puts "Restore the backup with: "
|
64
|
-
puts "mongorestore [--drop] --oplogReplay #{File.join(dir, 'dump')}"
|
66
|
+
puts "mongorestore [--drop] [--gzip] --oplogReplay #{File.join(dir, 'dump')}"
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
@@ -53,9 +53,10 @@ module MongoOplogBackup
|
|
53
53
|
|
54
54
|
query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]
|
55
55
|
|
56
|
-
|
57
|
-
|
58
|
-
|
56
|
+
dump_args = ['--out', config.oplog_dump_folder, '--db', 'local', '--collection', 'oplog.rs']
|
57
|
+
dump_args += query
|
58
|
+
dump_args << '--gzip' if config.use_compression?
|
59
|
+
config.mongodump(dump_args)
|
59
60
|
|
60
61
|
unless File.exists? config.oplog_dump
|
61
62
|
raise "mongodump failed"
|
@@ -89,6 +90,7 @@ module MongoOplogBackup
|
|
89
90
|
result[:empty] = true
|
90
91
|
else
|
91
92
|
outfile = "oplog-#{first}-#{last}.bson"
|
93
|
+
outfile += '.gz' if config.use_compression?
|
92
94
|
full_path = File.join(backup_folder, outfile)
|
93
95
|
FileUtils.mkdir_p backup_folder
|
94
96
|
FileUtils.mv config.oplog_dump, full_path
|
@@ -124,7 +126,9 @@ module MongoOplogBackup
|
|
124
126
|
raise "Backup folder '#{backup_folder}' already exists; not performing backup."
|
125
127
|
end
|
126
128
|
dump_folder = File.join(backup_folder, 'dump')
|
127
|
-
|
129
|
+
dump_args = ['--out', dump_folder]
|
130
|
+
dump_args << '--gzip' if config.use_compression?
|
131
|
+
result = config.mongodump(dump_args)
|
128
132
|
unless File.directory? dump_folder
|
129
133
|
MongoOplogBackup.log.error 'Backup folder does not exist'
|
130
134
|
raise 'Full backup failed'
|
@@ -159,7 +163,7 @@ module MongoOplogBackup
|
|
159
163
|
end
|
160
164
|
|
161
165
|
if mode == :oplog
|
162
|
-
raise "Unknown backup position - cannot perform oplog backup." unless have_backup
|
166
|
+
raise "Unknown backup position - cannot perform oplog backup. Have you completed a full backup?" unless have_backup
|
163
167
|
MongoOplogBackup.log.info "Performing incremental oplog backup"
|
164
168
|
lock(File.join(backup_folder, 'backup.lock')) do
|
165
169
|
result = backup_oplog
|
@@ -14,6 +14,7 @@ module MongoOplogBackup
|
|
14
14
|
options = {}
|
15
15
|
unless file.nil?
|
16
16
|
conf = YAML.load_file(file)
|
17
|
+
options[:gzip] = conf["gzip"] unless conf["gzip"].nil?
|
17
18
|
options[:ssl] = conf["ssl"] unless conf["ssl"].nil?
|
18
19
|
options[:sslAllowInvalidCertificates] = conf["sslAllowInvalidCertificates"] unless conf["sslAllowInvalidCertificates"].nil?
|
19
20
|
options[:sslCAFile] = conf["sslCAFile"] unless conf["sslCAFile"].nil?
|
@@ -30,6 +31,10 @@ module MongoOplogBackup
|
|
30
31
|
options[:dir]
|
31
32
|
end
|
32
33
|
|
34
|
+
def use_compression?
|
35
|
+
!!options[:gzip]
|
36
|
+
end
|
37
|
+
|
33
38
|
def command_line_options
|
34
39
|
args = []
|
35
40
|
args << '--ssl' if options[:ssl]
|
@@ -48,7 +53,11 @@ module MongoOplogBackup
|
|
48
53
|
end
|
49
54
|
|
50
55
|
def oplog_dump
|
51
|
-
|
56
|
+
if use_compression?
|
57
|
+
File.join(oplog_dump_folder, 'local/oplog.rs.bson.gz')
|
58
|
+
else
|
59
|
+
File.join(oplog_dump_folder, 'local/oplog.rs.bson')
|
60
|
+
end
|
52
61
|
end
|
53
62
|
|
54
63
|
def global_state_file
|
@@ -1,11 +1,19 @@
|
|
1
|
+
require 'zlib'
|
1
2
|
module MongoOplogBackup
|
2
3
|
module Oplog
|
3
4
|
def self.each_document(filename)
|
4
|
-
|
5
|
+
yield_bson_document = Proc.new do |stream|
|
5
6
|
while !stream.eof?
|
7
|
+
# FIXME: Since bson4, from_bson takes a ByteArray instead of a StringIO
|
6
8
|
yield BSON::Document.from_bson(stream)
|
7
9
|
end
|
8
10
|
end
|
11
|
+
|
12
|
+
if gzip_fingerprint(filename)
|
13
|
+
Zlib::GzipReader.open(filename, &yield_bson_document)
|
14
|
+
else
|
15
|
+
File.open(filename, 'rb', &yield_bson_document)
|
16
|
+
end
|
9
17
|
end
|
10
18
|
|
11
19
|
def self.oplog_timestamps(filename)
|
@@ -18,7 +26,7 @@ module MongoOplogBackup
|
|
18
26
|
timestamps
|
19
27
|
end
|
20
28
|
|
21
|
-
FILENAME_RE = /\/oplog-(\d+):(\d+)-(\d+):(\d+)\.bson
|
29
|
+
FILENAME_RE = /\/oplog-(\d+):(\d+)-(\d+):(\d+)\.bson(?:\.gz)?\z/
|
22
30
|
|
23
31
|
def self.timestamps_from_filename filename
|
24
32
|
match = FILENAME_RE.match(filename)
|
@@ -38,8 +46,9 @@ module MongoOplogBackup
|
|
38
46
|
def self.merge(target, source_files, options={})
|
39
47
|
limit = options[:limit] # TODO: use
|
40
48
|
force = options[:force]
|
49
|
+
compress = !!options[:gzip]
|
41
50
|
|
42
|
-
|
51
|
+
process_output = Proc.new do |output|
|
43
52
|
last_timestamp = nil
|
44
53
|
first = true
|
45
54
|
|
@@ -65,6 +74,10 @@ module MongoOplogBackup
|
|
65
74
|
Oplog.each_document(filename) do |doc|
|
66
75
|
timestamp = doc['ts']
|
67
76
|
first_file_timestamp = timestamp if first_file_timestamp.nil?
|
77
|
+
|
78
|
+
# gzip stores the mtime in the header, so we set it explicity for consistency between runs.
|
79
|
+
output.mtime = first_file_timestamp.seconds if output.mtime.to_i == 0
|
80
|
+
|
68
81
|
if !last_timestamp.nil? && timestamp <= last_timestamp
|
69
82
|
skipped += 1
|
70
83
|
elsif !last_file_timestamp.nil? && timestamp <= last_file_timestamp
|
@@ -90,10 +103,15 @@ module MongoOplogBackup
|
|
90
103
|
first = false
|
91
104
|
end
|
92
105
|
end
|
106
|
+
if (compress)
|
107
|
+
Zlib::GzipWriter.open(target, &process_output)
|
108
|
+
else
|
109
|
+
File.open(target, 'wb', &process_output)
|
110
|
+
end
|
93
111
|
end
|
94
112
|
|
95
113
|
def self.find_oplogs(dir)
|
96
|
-
files = Dir.glob(File.join(dir, 'oplog-*.bson'))
|
114
|
+
files = Dir.glob(File.join(dir, 'oplog-*.bson*'))
|
97
115
|
files.keep_if {|name| name =~ FILENAME_RE}
|
98
116
|
files.sort! {|a, b| timestamps_from_filename(a)[:first] <=> timestamps_from_filename(b)[:first]}
|
99
117
|
files
|
@@ -101,9 +119,15 @@ module MongoOplogBackup
|
|
101
119
|
|
102
120
|
def self.merge_backup(dir)
|
103
121
|
oplogs = find_oplogs(dir)
|
104
|
-
|
122
|
+
compress_target = oplogs.any? { |o| o.end_with?('.gz') }
|
123
|
+
target = File.join(dir, 'dump', 'oplog.bson') # Mongorestore expects this filename, without a gzip suffix.
|
105
124
|
FileUtils.mkdir_p(File.join(dir, 'dump'))
|
106
|
-
merge(target, oplogs)
|
125
|
+
merge(target, oplogs, {gzip: compress_target})
|
126
|
+
end
|
127
|
+
|
128
|
+
def self.gzip_fingerprint filename
|
129
|
+
bytes = File.read(filename, 2, 0)
|
130
|
+
bytes[0] == "\x1f".force_encoding('BINARY') && bytes[1] == "\x8b".force_encoding('BINARY')
|
107
131
|
end
|
108
132
|
|
109
133
|
end
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/spec/oplog_spec.rb
CHANGED
@@ -55,4 +55,58 @@ describe MongoOplogBackup::Oplog do
|
|
55
55
|
|
56
56
|
'spec-tmp/backup/dump/oplog.bson'.should be_same_oplog_as oplog_merged
|
57
57
|
end
|
58
|
+
|
59
|
+
|
60
|
+
|
61
|
+
context 'with gzipped oplogs' do
|
62
|
+
let(:oplog1) { 'spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz'}
|
63
|
+
let(:oplog2) { 'spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz'}
|
64
|
+
let(:oplog3) { 'spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz'}
|
65
|
+
let(:oplog_merged) { 'spec/fixtures/gzip/oplog-merged-gzipped.bson'}
|
66
|
+
|
67
|
+
it 'should extract oplog timestamps' do
|
68
|
+
timestamps = MongoOplogBackup::Oplog.oplog_timestamps(oplog1)
|
69
|
+
timestamps.should == [
|
70
|
+
BSON::Timestamp.new(1479827504, 7),
|
71
|
+
BSON::Timestamp.new(1479827515, 1),
|
72
|
+
BSON::Timestamp.new(1479827517, 1),
|
73
|
+
BSON::Timestamp.new(1479827518, 1)
|
74
|
+
]
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'should merge oplogs' do
|
78
|
+
merged_out = 'spec-tmp/oplog-merged-gzipped.bson'
|
79
|
+
MongoOplogBackup::Oplog.merge(merged_out, [oplog1, oplog2, oplog3], {gzip: true})
|
80
|
+
|
81
|
+
expected_timestamps =
|
82
|
+
MongoOplogBackup::Oplog.oplog_timestamps(oplog1) +
|
83
|
+
MongoOplogBackup::Oplog.oplog_timestamps(oplog2) +
|
84
|
+
MongoOplogBackup::Oplog.oplog_timestamps(oplog3)
|
85
|
+
|
86
|
+
expected_timestamps.uniq!
|
87
|
+
expected_timestamps.sort! # Not sure if uniq! modifies the order
|
88
|
+
|
89
|
+
actual_timestamps = MongoOplogBackup::Oplog.oplog_timestamps(merged_out)
|
90
|
+
actual_timestamps.should == expected_timestamps
|
91
|
+
|
92
|
+
merged_out.should be_same_oplog_as oplog_merged
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'should parse timestamps from a filename' do
|
96
|
+
timestamps = MongoOplogBackup::Oplog.timestamps_from_filename('some/oplog-1408088734:1-1408088740:52.bson.gz')
|
97
|
+
timestamps.should == {
|
98
|
+
first: BSON::Timestamp.new(1408088734, 1),
|
99
|
+
last: BSON::Timestamp.new(1408088740, 52)
|
100
|
+
}
|
101
|
+
end
|
102
|
+
|
103
|
+
it "should merge a backup folder" do
|
104
|
+
FileUtils.mkdir_p 'spec-tmp/backup-zipped'
|
105
|
+
FileUtils.cp_r Dir['spec/fixtures/gzip/oplog-*.bson.gz'], 'spec-tmp/backup-zipped/'
|
106
|
+
|
107
|
+
MongoOplogBackup::Oplog.merge_backup('spec-tmp/backup-zipped')
|
108
|
+
|
109
|
+
'spec-tmp/backup-zipped/dump/oplog.bson'.should be_same_oplog_as oplog_merged
|
110
|
+
end
|
111
|
+
end
|
58
112
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongo-oplog-backup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ralf Kistner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bson
|
@@ -125,6 +125,10 @@ files:
|
|
125
125
|
- spec/backup_spec.rb
|
126
126
|
- spec/command_spec.rb
|
127
127
|
- spec/enumerable_spec.rb
|
128
|
+
- spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz
|
129
|
+
- spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz
|
130
|
+
- spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz
|
131
|
+
- spec/fixtures/gzip/oplog-merged-gzipped.bson
|
128
132
|
- spec/fixtures/oplog-1408088734:1-1408088740:1.bson
|
129
133
|
- spec/fixtures/oplog-1408088740:1-1408088810:1.bson
|
130
134
|
- spec/fixtures/oplog-1408088810:1-1408088928:1.bson
|
@@ -160,6 +164,10 @@ test_files:
|
|
160
164
|
- spec/backup_spec.rb
|
161
165
|
- spec/command_spec.rb
|
162
166
|
- spec/enumerable_spec.rb
|
167
|
+
- spec/fixtures/gzip/oplog-1479827504:7-1479827518:1.bson.gz
|
168
|
+
- spec/fixtures/gzip/oplog-1479827518:1-1479827535:1.bson.gz
|
169
|
+
- spec/fixtures/gzip/oplog-1479827535:1-1479828312:1.bson.gz
|
170
|
+
- spec/fixtures/gzip/oplog-merged-gzipped.bson
|
163
171
|
- spec/fixtures/oplog-1408088734:1-1408088740:1.bson
|
164
172
|
- spec/fixtures/oplog-1408088740:1-1408088810:1.bson
|
165
173
|
- spec/fixtures/oplog-1408088810:1-1408088928:1.bson
|