mongo-oplog-backup 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/mongo_oplog_backup.rb +2 -0
- data/lib/mongo_oplog_backup/backup.rb +10 -6
- data/lib/mongo_oplog_backup/command.rb +124 -0
- data/lib/mongo_oplog_backup/config.rb +31 -14
- data/lib/mongo_oplog_backup/version.rb +1 -1
- data/spec/command_spec.rb +33 -0
- metadata +4 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6bad50ae17686f3dadca7ed09faa925433f2ba4f
|
4
|
+
data.tar.gz: eb9e08a62cdddf9b0490d4903f62da08b47e19cf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 44a3a86eab76a36be2a0d25eda220be4a12b4d597987f5e8221da365aa11d17d58db0b8e848be588cf161654a9f944cfd1e800c0e1def211b9df779b0195e35a
|
7
|
+
data.tar.gz: a79050e079b5f40a48ea2865044aa075aa2f10c91b53191ff78aa2a604bb7ee95efa41189cda21ee7f06c12c0f458026cdded37906eb131842270cba42445b19
|
data/lib/mongo_oplog_backup.rb
CHANGED
@@ -4,6 +4,7 @@ require 'mongo_oplog_backup/version'
|
|
4
4
|
require 'mongo_oplog_backup/ext/enumerable'
|
5
5
|
require 'mongo_oplog_backup/ext/timestamp'
|
6
6
|
|
7
|
+
require 'mongo_oplog_backup/command'
|
7
8
|
require 'mongo_oplog_backup/config'
|
8
9
|
require 'mongo_oplog_backup/backup'
|
9
10
|
require 'mongo_oplog_backup/oplog'
|
@@ -15,6 +16,7 @@ module MongoOplogBackup
|
|
15
16
|
|
16
17
|
def self.log= log
|
17
18
|
@@log = log
|
19
|
+
Command.logger = log
|
18
20
|
end
|
19
21
|
|
20
22
|
@@log = Logger.new STDOUT
|
@@ -30,11 +30,13 @@ module MongoOplogBackup
|
|
30
30
|
raise ArgumentError, ":start is required" unless start_at
|
31
31
|
|
32
32
|
if start_at
|
33
|
-
query =
|
33
|
+
query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]
|
34
34
|
else
|
35
|
-
query =
|
35
|
+
query = []
|
36
36
|
end
|
37
|
-
config.mongodump(
|
37
|
+
config.mongodump(['--out', config.oplog_dump_folder,
|
38
|
+
'--db', 'local', '--collection', 'oplog.rs'] +
|
39
|
+
query)
|
38
40
|
|
39
41
|
unless File.exists? config.oplog_dump
|
40
42
|
raise "mongodump failed"
|
@@ -82,7 +84,7 @@ module MongoOplogBackup
|
|
82
84
|
|
83
85
|
def latest_oplog_timestamp
|
84
86
|
script = File.expand_path('../../oplog-last-timestamp.js', File.dirname(__FILE__))
|
85
|
-
result_text = config.mongo('admin', script)
|
87
|
+
result_text = config.mongo('admin', script).standard_output
|
86
88
|
begin
|
87
89
|
response = JSON.parse(result_text)
|
88
90
|
return nil unless response['position']
|
@@ -97,8 +99,10 @@ module MongoOplogBackup
|
|
97
99
|
raise "Cannot backup with empty oplog" if position.nil?
|
98
100
|
backup_name = "backup-#{position}"
|
99
101
|
dump_folder = File.join(config.backup_dir, backup_name, 'dump')
|
100
|
-
|
101
|
-
|
102
|
+
config.mongodump('--out', dump_folder)
|
103
|
+
unless File.directory? dump_folder
|
104
|
+
raise 'Full backup failed'
|
105
|
+
end
|
102
106
|
return {
|
103
107
|
position: position,
|
104
108
|
backup: backup_name
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'open3'
|
2
|
+
require 'io/wait'
|
3
|
+
|
4
|
+
|
5
|
+
module MongoOplogBackup
|
6
|
+
class Command
|
7
|
+
def self.logger= logger
|
8
|
+
@logger = logger
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.logger
|
12
|
+
@logger
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :command
|
16
|
+
attr_reader :standard_output
|
17
|
+
attr_reader :standard_error
|
18
|
+
attr_reader :status
|
19
|
+
|
20
|
+
def self.execute(command, options={})
|
21
|
+
Command.new(command, options).run
|
22
|
+
end
|
23
|
+
|
24
|
+
# command must be an array containing the command and arguments
|
25
|
+
def initialize(command, options={})
|
26
|
+
@command = command
|
27
|
+
@standard_output = ''
|
28
|
+
@standard_error = ''
|
29
|
+
@out_blocks = []
|
30
|
+
@err_blocks = []
|
31
|
+
|
32
|
+
logger = options[:logger] || Command.logger
|
33
|
+
|
34
|
+
if logger
|
35
|
+
log_output(logger)
|
36
|
+
end
|
37
|
+
|
38
|
+
on_stdout do |data|
|
39
|
+
@standard_output << data
|
40
|
+
end
|
41
|
+
on_stderr do |data|
|
42
|
+
@standard_error << data
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def on_stdout_line &block
|
47
|
+
on_stdout(&lines_proc(&block))
|
48
|
+
end
|
49
|
+
|
50
|
+
def on_stderr_line &block
|
51
|
+
on_stderr(&lines_proc(&block))
|
52
|
+
end
|
53
|
+
|
54
|
+
def on_stderr &block
|
55
|
+
@err_blocks << block
|
56
|
+
end
|
57
|
+
|
58
|
+
def on_stdout &block
|
59
|
+
@out_blocks << block
|
60
|
+
end
|
61
|
+
|
62
|
+
def log_output(logger)
|
63
|
+
on_stdout_line do |line|
|
64
|
+
logger.debug(line)
|
65
|
+
end
|
66
|
+
on_stderr_line do |line|
|
67
|
+
logger.error(line)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def run
|
72
|
+
@status = Open3.popen3(*command) do |stdin, stdout, stderr, wait_thr|
|
73
|
+
stdin.close_write
|
74
|
+
until all_eof([stdout, stderr])
|
75
|
+
read_available_data(stdout) do |data|
|
76
|
+
@out_blocks.each do |block|
|
77
|
+
block.call(data)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
read_available_data(stderr) do |data|
|
81
|
+
@err_blocks.each do |block|
|
82
|
+
block.call(data)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
sleep 0.001
|
86
|
+
end
|
87
|
+
|
88
|
+
wait_thr.value
|
89
|
+
end
|
90
|
+
raise!
|
91
|
+
self
|
92
|
+
end
|
93
|
+
|
94
|
+
def raise!
|
95
|
+
unless status.success?
|
96
|
+
raise "Command failed with exit code #{status.exitstatus}"
|
97
|
+
end
|
98
|
+
self
|
99
|
+
end
|
100
|
+
|
101
|
+
private
|
102
|
+
def lines_proc &block
|
103
|
+
# TODO: buffer partial lines
|
104
|
+
return Proc.new do |data|
|
105
|
+
data.split("\n").each do |line|
|
106
|
+
block.call line
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
BLOCK_SIZE = 1024
|
112
|
+
|
113
|
+
def read_available_data(io, &block)
|
114
|
+
if io.ready?
|
115
|
+
data = io.read_nonblock(BLOCK_SIZE)
|
116
|
+
block.call data
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def all_eof(files)
|
121
|
+
files.find { |f| !f.eof }.nil?
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'shellwords'
|
2
|
+
|
1
3
|
module MongoOplogBackup
|
2
4
|
class Config
|
3
5
|
attr_reader :options
|
@@ -18,7 +20,7 @@ module MongoOplogBackup
|
|
18
20
|
options[:username] = conf["username"] unless conf["username"].nil?
|
19
21
|
options[:password] = conf["password"] unless conf["password"].nil?
|
20
22
|
end
|
21
|
-
|
23
|
+
|
22
24
|
options
|
23
25
|
end
|
24
26
|
|
@@ -27,14 +29,15 @@ module MongoOplogBackup
|
|
27
29
|
end
|
28
30
|
|
29
31
|
def command_line_options
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
32
|
+
args = []
|
33
|
+
args << '--ssl' if options[:ssl]
|
34
|
+
[:host, :port, :username, :password].each do |option|
|
35
|
+
args += ["--#{option}", options[option].strip] if options[option]
|
36
|
+
end
|
37
|
+
|
38
|
+
args += ['--authenticationDatabase', 'admin'] if options[:username]
|
39
|
+
|
40
|
+
args
|
38
41
|
end
|
39
42
|
|
40
43
|
def oplog_dump_folder
|
@@ -54,16 +57,30 @@ module MongoOplogBackup
|
|
54
57
|
end
|
55
58
|
|
56
59
|
def exec(cmd)
|
57
|
-
MongoOplogBackup.log.debug ">>> #{cmd}"
|
58
|
-
|
60
|
+
MongoOplogBackup.log.debug ">>> #{command_string(cmd)}"
|
61
|
+
Command.execute(cmd)
|
59
62
|
end
|
60
63
|
|
61
|
-
def mongodump(args)
|
62
|
-
|
64
|
+
def mongodump(*args)
|
65
|
+
exec(['mongodump'] + command_line_options + args.flatten)
|
63
66
|
end
|
64
67
|
|
65
68
|
def mongo(db, script)
|
66
|
-
exec(
|
69
|
+
exec(['mongo'] + command_line_options + ['--quiet', '--norc', script])
|
70
|
+
end
|
71
|
+
|
72
|
+
def command_string(cmd)
|
73
|
+
previous = nil
|
74
|
+
filtered = cmd.map do |token|
|
75
|
+
pwd = (previous == '--password')
|
76
|
+
previous = token
|
77
|
+
if pwd
|
78
|
+
'***'
|
79
|
+
else
|
80
|
+
token
|
81
|
+
end
|
82
|
+
end
|
83
|
+
filtered.shelljoin
|
67
84
|
end
|
68
85
|
end
|
69
86
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe MongoOplogBackup::Command do
|
4
|
+
it 'should get stdout' do
|
5
|
+
result = MongoOplogBackup::Command.execute(['echo', 'something'])
|
6
|
+
result.standard_output.should == "something\n"
|
7
|
+
result.standard_error.should == ""
|
8
|
+
result.status.exitstatus.should == 0
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should get stderr' do
|
12
|
+
result = MongoOplogBackup::Command.execute(['ruby', '-e', '$stderr.puts "FOO"'])
|
13
|
+
result.standard_output.should == ""
|
14
|
+
result.standard_error.should == "FOO\n"
|
15
|
+
result.status.exitstatus.should == 0
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should raise on a non-zero exit code' do
|
19
|
+
command = MongoOplogBackup::Command.new(['ruby', '-e', 'exit 123'])
|
20
|
+
-> { command.run }.should raise_error
|
21
|
+
command.status.exitstatus.should == 123
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'should log' do
|
25
|
+
io = StringIO.new
|
26
|
+
logger = Logger.new io
|
27
|
+
MongoOplogBackup::Command.execute(['ruby', '-e', 'puts "BAR"; $stderr.puts "FOO"'], logger: logger)
|
28
|
+
io.rewind
|
29
|
+
log = io.read
|
30
|
+
log.should =~ /D, \[.+\] DEBUG -- : BAR\nE, \[.+\] ERROR -- : FOO\n/
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mongo-oplog-backup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ralf Kistner
|
@@ -113,6 +113,7 @@ files:
|
|
113
113
|
- bin/mongo-oplog-backup
|
114
114
|
- lib/mongo_oplog_backup.rb
|
115
115
|
- lib/mongo_oplog_backup/backup.rb
|
116
|
+
- lib/mongo_oplog_backup/command.rb
|
116
117
|
- lib/mongo_oplog_backup/config.rb
|
117
118
|
- lib/mongo_oplog_backup/ext/enumerable.rb
|
118
119
|
- lib/mongo_oplog_backup/ext/timestamp.rb
|
@@ -122,6 +123,7 @@ files:
|
|
122
123
|
- oplog-last-timestamp.js
|
123
124
|
- sample-config.yml
|
124
125
|
- spec/backup_spec.rb
|
126
|
+
- spec/command_spec.rb
|
125
127
|
- spec/enumerable_spec.rb
|
126
128
|
- spec/fixtures/oplog-1408088734:1-1408088740:1.bson
|
127
129
|
- spec/fixtures/oplog-1408088740:1-1408088810:1.bson
|
@@ -156,6 +158,7 @@ specification_version: 4
|
|
156
158
|
summary: Incremental backups for MongoDB using the oplog.
|
157
159
|
test_files:
|
158
160
|
- spec/backup_spec.rb
|
161
|
+
- spec/command_spec.rb
|
159
162
|
- spec/enumerable_spec.rb
|
160
163
|
- spec/fixtures/oplog-1408088734:1-1408088740:1.bson
|
161
164
|
- spec/fixtures/oplog-1408088740:1-1408088810:1.bson
|