mongo-oplog-backup 0.0.4 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c464ac7e971978efe22788614317405be8935f25
4
- data.tar.gz: a12be71eeb235821321b8946ef6bcf10ff040724
3
+ metadata.gz: 6bad50ae17686f3dadca7ed09faa925433f2ba4f
4
+ data.tar.gz: eb9e08a62cdddf9b0490d4903f62da08b47e19cf
5
5
  SHA512:
6
- metadata.gz: 2e495dcd5d7ebf02816187845dd7424980e77b5f098828dee1155d88fe59649d196d342ad5de048adb7631dd4faf98cc798cc2bc0180fde46ae3be5beb78e64c
7
- data.tar.gz: accab78d844a1e729e0a1dc99264def3e07c31509344d337a32fe7bdb35c6f95288037bcc1e919004fba67514da4672fbbded878fdfe80885ccd365d35348789
6
+ metadata.gz: 44a3a86eab76a36be2a0d25eda220be4a12b4d597987f5e8221da365aa11d17d58db0b8e848be588cf161654a9f944cfd1e800c0e1def211b9df779b0195e35a
7
+ data.tar.gz: a79050e079b5f40a48ea2865044aa075aa2f10c91b53191ff78aa2a604bb7ee95efa41189cda21ee7f06c12c0f458026cdded37906eb131842270cba42445b19
@@ -4,6 +4,7 @@ require 'mongo_oplog_backup/version'
4
4
  require 'mongo_oplog_backup/ext/enumerable'
5
5
  require 'mongo_oplog_backup/ext/timestamp'
6
6
 
7
+ require 'mongo_oplog_backup/command'
7
8
  require 'mongo_oplog_backup/config'
8
9
  require 'mongo_oplog_backup/backup'
9
10
  require 'mongo_oplog_backup/oplog'
@@ -15,6 +16,7 @@ module MongoOplogBackup
15
16
 
16
17
  def self.log= log
17
18
  @@log = log
19
+ Command.logger = log
18
20
  end
19
21
 
20
22
  @@log = Logger.new STDOUT
@@ -30,11 +30,13 @@ module MongoOplogBackup
30
30
  raise ArgumentError, ":start is required" unless start_at
31
31
 
32
32
  if start_at
33
- query = "--query \"{ts : { \\$gte : { \\$timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}\""
33
+ query = ['--query', "{ts : { $gte : { $timestamp : { t : #{start_at.seconds}, i : #{start_at.increment} } } }}"]
34
34
  else
35
- query = ""
35
+ query = []
36
36
  end
37
- config.mongodump("--out #{config.oplog_dump_folder} --db local --collection oplog.rs #{query}")
37
+ config.mongodump(['--out', config.oplog_dump_folder,
38
+ '--db', 'local', '--collection', 'oplog.rs'] +
39
+ query)
38
40
 
39
41
  unless File.exists? config.oplog_dump
40
42
  raise "mongodump failed"
@@ -82,7 +84,7 @@ module MongoOplogBackup
82
84
 
83
85
  def latest_oplog_timestamp
84
86
  script = File.expand_path('../../oplog-last-timestamp.js', File.dirname(__FILE__))
85
- result_text = config.mongo('admin', script)
87
+ result_text = config.mongo('admin', script).standard_output
86
88
  begin
87
89
  response = JSON.parse(result_text)
88
90
  return nil unless response['position']
@@ -97,8 +99,10 @@ module MongoOplogBackup
97
99
  raise "Cannot backup with empty oplog" if position.nil?
98
100
  backup_name = "backup-#{position}"
99
101
  dump_folder = File.join(config.backup_dir, backup_name, 'dump')
100
- # TODO: fail hard if this command fails.
101
- config.mongodump("--out #{dump_folder}")
102
+ config.mongodump('--out', dump_folder)
103
+ unless File.directory? dump_folder
104
+ raise 'Full backup failed'
105
+ end
102
106
  return {
103
107
  position: position,
104
108
  backup: backup_name
@@ -0,0 +1,124 @@
1
+ require 'open3'
2
+ require 'io/wait'
3
+
4
+
5
+ module MongoOplogBackup
6
+ class Command
7
+ def self.logger= logger
8
+ @logger = logger
9
+ end
10
+
11
+ def self.logger
12
+ @logger
13
+ end
14
+
15
+ attr_reader :command
16
+ attr_reader :standard_output
17
+ attr_reader :standard_error
18
+ attr_reader :status
19
+
20
+ def self.execute(command, options={})
21
+ Command.new(command, options).run
22
+ end
23
+
24
+ # command must be an array containing the command and arguments
25
+ def initialize(command, options={})
26
+ @command = command
27
+ @standard_output = ''
28
+ @standard_error = ''
29
+ @out_blocks = []
30
+ @err_blocks = []
31
+
32
+ logger = options[:logger] || Command.logger
33
+
34
+ if logger
35
+ log_output(logger)
36
+ end
37
+
38
+ on_stdout do |data|
39
+ @standard_output << data
40
+ end
41
+ on_stderr do |data|
42
+ @standard_error << data
43
+ end
44
+ end
45
+
46
+ def on_stdout_line &block
47
+ on_stdout(&lines_proc(&block))
48
+ end
49
+
50
+ def on_stderr_line &block
51
+ on_stderr(&lines_proc(&block))
52
+ end
53
+
54
+ def on_stderr &block
55
+ @err_blocks << block
56
+ end
57
+
58
+ def on_stdout &block
59
+ @out_blocks << block
60
+ end
61
+
62
+ def log_output(logger)
63
+ on_stdout_line do |line|
64
+ logger.debug(line)
65
+ end
66
+ on_stderr_line do |line|
67
+ logger.error(line)
68
+ end
69
+ end
70
+
71
+ def run
72
+ @status = Open3.popen3(*command) do |stdin, stdout, stderr, wait_thr|
73
+ stdin.close_write
74
+ until all_eof([stdout, stderr])
75
+ read_available_data(stdout) do |data|
76
+ @out_blocks.each do |block|
77
+ block.call(data)
78
+ end
79
+ end
80
+ read_available_data(stderr) do |data|
81
+ @err_blocks.each do |block|
82
+ block.call(data)
83
+ end
84
+ end
85
+ sleep 0.001
86
+ end
87
+
88
+ wait_thr.value
89
+ end
90
+ raise!
91
+ self
92
+ end
93
+
94
+ def raise!
95
+ unless status.success?
96
+ raise "Command failed with exit code #{status.exitstatus}"
97
+ end
98
+ self
99
+ end
100
+
101
+ private
102
+ def lines_proc &block
103
+ # TODO: buffer partial lines
104
+ return Proc.new do |data|
105
+ data.split("\n").each do |line|
106
+ block.call line
107
+ end
108
+ end
109
+ end
110
+
111
+ BLOCK_SIZE = 1024
112
+
113
+ def read_available_data(io, &block)
114
+ if io.ready?
115
+ data = io.read_nonblock(BLOCK_SIZE)
116
+ block.call data
117
+ end
118
+ end
119
+
120
+ def all_eof(files)
121
+ files.find { |f| !f.eof }.nil?
122
+ end
123
+ end
124
+ end
@@ -1,3 +1,5 @@
1
+ require 'shellwords'
2
+
1
3
  module MongoOplogBackup
2
4
  class Config
3
5
  attr_reader :options
@@ -18,7 +20,7 @@ module MongoOplogBackup
18
20
  options[:username] = conf["username"] unless conf["username"].nil?
19
21
  options[:password] = conf["password"] unless conf["password"].nil?
20
22
  end
21
-
23
+
22
24
  options
23
25
  end
24
26
 
@@ -27,14 +29,15 @@ module MongoOplogBackup
27
29
  end
28
30
 
29
31
  def command_line_options
30
- ssl = options[:ssl] ? '--ssl ' : ''
31
- host = options[:host] ? "--host #{options[:host].strip} " : ''
32
- port = options[:port] ? "--port #{options[:port].strip} " : ''
33
- username = options[:username] ? "--username #{options[:username].strip} " : ''
34
- password = options[:password] ? "--password #{options[:password].strip} " : ''
35
- # TODO: make this configurable?
36
- authdb = options[:username] ? '--authenticationDatabase admin ' : ''
37
- "#{host}#{port}#{ssl}#{username}#{password}"
32
+ args = []
33
+ args << '--ssl' if options[:ssl]
34
+ [:host, :port, :username, :password].each do |option|
35
+ args += ["--#{option}", options[option].strip] if options[option]
36
+ end
37
+
38
+ args += ['--authenticationDatabase', 'admin'] if options[:username]
39
+
40
+ args
38
41
  end
39
42
 
40
43
  def oplog_dump_folder
@@ -54,16 +57,30 @@ module MongoOplogBackup
54
57
  end
55
58
 
56
59
  def exec(cmd)
57
- MongoOplogBackup.log.debug ">>> #{cmd}"
58
- `#{cmd}`
60
+ MongoOplogBackup.log.debug ">>> #{command_string(cmd)}"
61
+ Command.execute(cmd)
59
62
  end
60
63
 
61
- def mongodump(args)
62
- MongoOplogBackup.log.info exec("mongodump #{command_line_options} #{args}")
64
+ def mongodump(*args)
65
+ exec(['mongodump'] + command_line_options + args.flatten)
63
66
  end
64
67
 
65
68
  def mongo(db, script)
66
- exec("mongo #{command_line_options} --quiet --norc #{db} #{script}")
69
+ exec(['mongo'] + command_line_options + ['--quiet', '--norc', script])
70
+ end
71
+
72
+ def command_string(cmd)
73
+ previous = nil
74
+ filtered = cmd.map do |token|
75
+ pwd = (previous == '--password')
76
+ previous = token
77
+ if pwd
78
+ '***'
79
+ else
80
+ token
81
+ end
82
+ end
83
+ filtered.shelljoin
67
84
  end
68
85
  end
69
86
  end
@@ -1,3 +1,3 @@
1
1
  module MongoOplogBackup
2
- VERSION = "0.0.4"
2
+ VERSION = "0.0.5"
3
3
  end
@@ -0,0 +1,33 @@
1
+ require 'spec_helper'
2
+
3
+ describe MongoOplogBackup::Command do
4
+ it 'should get stdout' do
5
+ result = MongoOplogBackup::Command.execute(['echo', 'something'])
6
+ result.standard_output.should == "something\n"
7
+ result.standard_error.should == ""
8
+ result.status.exitstatus.should == 0
9
+ end
10
+
11
+ it 'should get stderr' do
12
+ result = MongoOplogBackup::Command.execute(['ruby', '-e', '$stderr.puts "FOO"'])
13
+ result.standard_output.should == ""
14
+ result.standard_error.should == "FOO\n"
15
+ result.status.exitstatus.should == 0
16
+ end
17
+
18
+ it 'should raise on a non-zero exit code' do
19
+ command = MongoOplogBackup::Command.new(['ruby', '-e', 'exit 123'])
20
+ -> { command.run }.should raise_error
21
+ command.status.exitstatus.should == 123
22
+ end
23
+
24
+ it 'should log' do
25
+ io = StringIO.new
26
+ logger = Logger.new io
27
+ MongoOplogBackup::Command.execute(['ruby', '-e', 'puts "BAR"; $stderr.puts "FOO"'], logger: logger)
28
+ io.rewind
29
+ log = io.read
30
+ log.should =~ /D, \[.+\] DEBUG -- : BAR\nE, \[.+\] ERROR -- : FOO\n/
31
+ end
32
+
33
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mongo-oplog-backup
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ralf Kistner
@@ -113,6 +113,7 @@ files:
113
113
  - bin/mongo-oplog-backup
114
114
  - lib/mongo_oplog_backup.rb
115
115
  - lib/mongo_oplog_backup/backup.rb
116
+ - lib/mongo_oplog_backup/command.rb
116
117
  - lib/mongo_oplog_backup/config.rb
117
118
  - lib/mongo_oplog_backup/ext/enumerable.rb
118
119
  - lib/mongo_oplog_backup/ext/timestamp.rb
@@ -122,6 +123,7 @@ files:
122
123
  - oplog-last-timestamp.js
123
124
  - sample-config.yml
124
125
  - spec/backup_spec.rb
126
+ - spec/command_spec.rb
125
127
  - spec/enumerable_spec.rb
126
128
  - spec/fixtures/oplog-1408088734:1-1408088740:1.bson
127
129
  - spec/fixtures/oplog-1408088740:1-1408088810:1.bson
@@ -156,6 +158,7 @@ specification_version: 4
156
158
  summary: Incremental backups for MongoDB using the oplog.
157
159
  test_files:
158
160
  - spec/backup_spec.rb
161
+ - spec/command_spec.rb
159
162
  - spec/enumerable_spec.rb
160
163
  - spec/fixtures/oplog-1408088734:1-1408088740:1.bson
161
164
  - spec/fixtures/oplog-1408088740:1-1408088810:1.bson