fluent-plugin-mongo 0.7.16 → 0.8.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,69 +0,0 @@
1
- require 'test_helper'
2
- require 'fluent/plugin/out_mongo'
3
-
4
- class MongoTagCollectionTest < Test::Unit::TestCase
5
- def setup
6
- Fluent::Test.setup
7
- end
8
-
9
- CONFIG = %[
10
- type mongo
11
- database fluent
12
- tag_mapped
13
- remove_tag_prefix should.remove.
14
- ]
15
-
16
- def create_driver(conf = CONFIG)
17
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput) {
18
- def start
19
- super
20
- end
21
-
22
- def shutdown
23
- super
24
- end
25
-
26
- def operate(collection, records)
27
- [format_collection_name(collection), records]
28
- end
29
-
30
- def get_or_create_collection(collection_name)
31
- collection_name
32
- end
33
-
34
- def mongod_version
35
- "2.0.0"
36
- end
37
- }.configure(conf)
38
- end
39
-
40
- def test_configure
41
- d = create_driver(CONFIG)
42
- assert_equal(/^should\.remove\./, d.instance.instance_variable_get(:@remove_tag_prefix))
43
- end
44
-
45
- def emit_documents(d)
46
- time = Time.parse("2011-01-02 13:14:15 UTC").to_i
47
- d.emit({'a' => 1}, time)
48
- d.emit({'a' => 2}, time)
49
- time
50
- end
51
-
52
- def test_write
53
- omit('BufferedOutputTestDriver should support emit arguments(chain and key)')
54
-
55
- d = create_driver(CONFIG)
56
- d.tag = 'mytag'
57
- t = emit_documents(d)
58
- mock(d.instance).operate('mytag', [{'a' => 1, d.instance.time_key => Time.at(t)},
59
- {'a' => 2, d.instance.time_key => Time.at(t)}])
60
- d.run
61
- end
62
-
63
- def test_remove_prefix_collection
64
- d = create_driver(CONFIG)
65
- assert_equal('prefix', d.instance.__send__(:format_collection_name, 'should.remove.prefix'))
66
- assert_equal('test', d.instance.__send__(:format_collection_name, '..test..'))
67
- assert_equal('test.foo', d.instance.__send__(:format_collection_name, '..test.foo.'))
68
- end
69
- end
data/test/test_helper.rb DELETED
@@ -1,59 +0,0 @@
1
- $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
- $LOAD_PATH.unshift(File.dirname(__FILE__))
3
-
4
- require 'rr'
5
- require 'test/unit'
6
-
7
- if ENV['SIMPLE_COV']
8
- require 'simplecov'
9
- SimpleCov.start do
10
- add_filter 'test/'
11
- add_filter 'pkg/'
12
- add_filter 'vendor/'
13
- end
14
- end
15
-
16
- require 'test/unit'
17
- require 'fluent/test'
18
-
19
- # for testing
20
-
21
- def unused_port
22
- s = TCPServer.open(0)
23
- port = s.addr[1]
24
- s.close
25
- port
26
- end
27
-
28
- # for MongoDB
29
-
30
- require 'mongo'
31
-
32
- MONGO_DB_DB = 'fluent_test'
33
- MONGO_DB_PATH = File.join(File.dirname(__FILE__), 'plugin', 'data')
34
-
35
- module MongoTestHelper
36
- def self.cleanup_mongod_env
37
- Process.kill "TERM", @@pid
38
- Process.waitpid @@pid
39
- system("rm -rf #{MONGO_DB_PATH}")
40
- end
41
-
42
- def self.setup_mongod
43
- system("rm -rf #{MONGO_DB_PATH}")
44
- system("mkdir -p #{MONGO_DB_PATH}")
45
-
46
- @@mongod_port = unused_port
47
- @@pid = spawn(ENV['mongod'], "--port=#{@@mongod_port}", "--dbpath=#{MONGO_DB_PATH}")
48
- sleep 3
49
- end
50
-
51
- def self.teardown_mongod
52
- Mongo::Connection.new('localhost', @@mongod_port).drop_database(MONGO_DB_DB)
53
- cleanup_mongod_env
54
- end
55
-
56
- def self.mongod_port
57
- @@mongod_port
58
- end
59
- end
@@ -1,14 +0,0 @@
1
- require File.join((File.expand_path(File.dirname(__FILE__))), 'repl_set_manager')
2
-
3
- class AuthReplSetManager < ReplSetManager
4
- def initialize(opts={})
5
- super(opts)
6
-
7
- @key_path = opts[:key_path] || File.join(File.expand_path(File.dirname(__FILE__)), "keyfile.txt")
8
- system("chmod 600 #{@key_path}")
9
- end
10
-
11
- def start_cmd(n)
12
- super + " --keyFile #{@key_path}"
13
- end
14
- end
@@ -1,420 +0,0 @@
1
- require 'thread'
2
-
3
- STDOUT.sync = true
4
-
5
- require 'mongo'
6
- #unless defined? Mongo
7
- # require File.join(File.dirname(__FILE__), '..', '..', 'lib', 'mongo')
8
- #end
9
-
10
- class ReplSetManager
11
-
12
- attr_accessor :host, :start_port, :ports, :name, :mongods, :tags, :version
13
-
14
- def initialize(opts={})
15
- @mongod = ENV['mongod'] || 'mongod'
16
- @start_port = opts[:start_port] || 30000
17
- @ports = []
18
- @name = opts[:name] || 'replica-set-foo'
19
- @host = opts[:host] || 'localhost'
20
- @retries = opts[:retries] || 30
21
- @config = {"_id" => @name, "members" => []}
22
- @durable = opts.fetch(:durable, false)
23
- @smallfiles = opts.fetch(:smallfiles, true)
24
- @prealloc = opts.fetch(:prealloc, false)
25
- @path = File.join(File.expand_path(File.dirname(__FILE__)), "data")
26
- @oplog_size = opts.fetch(:oplog_size, 16)
27
- @tags = [{"dc" => "ny", "rack" => "a", "db" => "main"},
28
- {"dc" => "ny", "rack" => "b", "db" => "main"},
29
- {"dc" => "sf", "rack" => "a", "db" => "main"}]
30
-
31
- @arbiter_count = opts[:arbiter_count] || 0
32
- @secondary_count = opts[:secondary_count] || 2
33
- @passive_count = opts[:passive_count] || 0
34
- @primary_count = 1
35
-
36
- @count = @primary_count + @passive_count + @arbiter_count + @secondary_count
37
- if @count > 7
38
- raise StandardError, "Cannot create a replica set with #{node_count} nodes. 7 is the max."
39
- end
40
-
41
- @mongods = {}
42
- version_string = `#{@mongod} --version`
43
- version_string =~ /(\d\.\d\.\d)/
44
- @version = $1.split(".").map {|d| d.to_i }
45
- end
46
-
47
- def start_set
48
- @mongods.keys.each do |node|
49
- kill(node)
50
- end
51
- should_start = true
52
- puts "** Starting a replica set with #{@count} nodes"
53
-
54
- n = 0
55
- (@primary_count + @secondary_count).times do
56
- init_node(n, should_start) do |attrs|
57
- if @version[0] >= 2
58
- attrs['tags'] = @tags[n % @tags.size]
59
- end
60
- end
61
- n += 1
62
- end
63
-
64
- @passive_count.times do
65
- init_node(n, should_start) do |attrs|
66
- attrs['priority'] = 0
67
- end
68
- n += 1
69
- end
70
-
71
- @arbiter_count.times do
72
- init_node(n, should_start) do |attrs|
73
- attrs['arbiterOnly'] = true
74
- end
75
- n += 1
76
- end
77
-
78
- initiate
79
- ensure_up
80
- end
81
-
82
- def cleanup_set
83
- @mongods.keys.each do |node|
84
- kill(node)
85
- end
86
- @count.times do |n|
87
- system("rm -rf #{@mongods[n]['db_path']}")
88
- end
89
- end
90
-
91
- def init_node(n, should_start=true)
92
- @mongods[n] ||= {}
93
- port = @start_port + n
94
- @ports << port
95
- @mongods[n]['port'] = port
96
- @mongods[n]['db_path'] = get_path("rs-#{port}")
97
- @mongods[n]['log_path'] = get_path("log-#{port}")
98
- @mongods[n]['start'] = start_cmd(n)
99
-
100
- if should_start
101
- system("rm -rf #{@mongods[n]['db_path']}")
102
- system("mkdir -p #{@mongods[n]['db_path']}")
103
- start(n)
104
- end
105
-
106
- member = {'_id' => n, 'host' => "#{@host}:#{@mongods[n]['port']}"}
107
-
108
- if block_given?
109
- custom_attrs = {}
110
- yield custom_attrs
111
- member.merge!(custom_attrs)
112
- @mongods[n].merge!(custom_attrs)
113
- end
114
-
115
- @config['members'] << member
116
- end
117
-
118
- def journal_switch
119
- if @version[0] >= 2
120
- if @durable
121
- "--journal"
122
- else
123
- "--nojournal"
124
- end
125
- elsif @durable
126
- "--journal"
127
- end
128
- end
129
-
130
- def start_cmd(n)
131
- @mongods[n]['start'] = "#{@mongod} --replSet #{@name} --logpath '#{@mongods[n]['log_path']}' " +
132
- "--oplogSize #{@oplog_size} #{journal_switch} --dbpath #{@mongods[n]['db_path']} --port #{@mongods[n]['port']} --fork"
133
- @mongods[n]['start'] += " --dur" if @durable
134
- @mongods[n]['start'] += " --smallfiles" if @smallfiles
135
- @mongods[n]['start'] += " --noprealloc" unless @prealloc
136
- @mongods[n]['start']
137
- end
138
-
139
- def remove_secondary_node
140
- primary = get_node_with_state(1)
141
- con = get_connection(primary)
142
- config = con['local']['system.replset'].find_one
143
- secondary = get_node_with_state(2)
144
- host_port = "#{@host}:#{@mongods[secondary]['port']}"
145
- kill(secondary)
146
- @mongods.delete(secondary)
147
- @config['members'].reject! {|m| m['host'] == host_port}
148
- @config['version'] = config['version'] + 1
149
-
150
- begin
151
- con['admin'].command({'replSetReconfig' => @config})
152
- rescue Mongo::ConnectionFailure
153
- end
154
-
155
- con.close
156
-
157
- return secondary
158
- end
159
-
160
- def add_node(n=nil, &block)
161
- primary = get_node_with_state(1)
162
- con = get_connection(primary)
163
-
164
- init_node(n || @mongods.length, &block)
165
- config = con['local']['system.replset'].find_one
166
- @config['version'] = config['version'] + 1
167
-
168
- # We expect a connection failure on reconfigure here.
169
- begin
170
- con['admin'].command({'replSetReconfig' => @config})
171
- rescue Mongo::ConnectionFailure
172
- end
173
-
174
- con.close
175
- ensure_up
176
- end
177
-
178
- def add_arbiter
179
- add_node do |attrs|
180
- attrs['arbiterOnly'] = true
181
- end
182
- end
183
-
184
- def wait_for_death(pid)
185
- @retries.times do
186
- if `ps a | grep mongod`.include?("#{pid}")
187
- puts "waiting for mongod @ pid #{pid} to die..."
188
- sleep(1)
189
- else
190
- puts "mongod @ pid #{pid} was killed successfully"
191
- return true
192
- end
193
- end
194
- puts "mongod never died"
195
- return false
196
- end
197
-
198
- def kill(node, signal=2)
199
- return unless @mongods[node]['up']
200
-
201
- pid = @mongods[node]['pid']
202
- puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
203
- Process.kill signal, pid
204
- dead = wait_for_death(pid)
205
- @mongods[node]['up'] = false if dead
206
- end
207
-
208
- def kill_primary(signal=2)
209
- node = get_node_with_state(1)
210
- kill(node, signal)
211
- return node
212
- end
213
-
214
- # Note that we have to rescue a connection failure
215
- # when we run the StepDown command because that
216
- # command will close the connection.
217
- def step_down_primary
218
- primary = get_node_with_state(1)
219
- con = get_connection(primary)
220
- begin
221
- con['admin'].command({'replSetStepDown' => 90})
222
- rescue Mongo::ConnectionFailure
223
- end
224
- con.close
225
- end
226
-
227
- def kill_secondary
228
- node = get_node_with_state(2)
229
- kill(node)
230
- return node
231
- end
232
-
233
- def kill_all_secondaries
234
- nodes = get_all_nodes_with_state(2)
235
- if nodes
236
- nodes.each do |n|
237
- kill(n)
238
- end
239
- end
240
- end
241
-
242
- def restart_killed_nodes
243
- nodes = @mongods.keys.select do |key|
244
- @mongods[key]['up'] == false
245
- end
246
-
247
- nodes.each do |node|
248
- start(node)
249
- end
250
-
251
- ensure_up
252
- end
253
-
254
- def get_node_from_port(port)
255
- @mongods.keys.detect { |key| @mongods[key]['port'] == port }
256
- end
257
-
258
- def start(node)
259
- system(@mongods[node]['start'])
260
- @mongods[node]['up'] = true
261
- sleep(0.5)
262
- @mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip.to_i
263
- end
264
- alias :restart :start
265
-
266
- def ensure_up(n=nil, connection=nil)
267
- print "** Ensuring members are up..."
268
-
269
- attempt(n) do
270
- print "."
271
- con = connection || get_connection
272
- begin
273
- status = con['admin'].command({:replSetGetStatus => 1})
274
- rescue Mongo::OperationFailure => ex
275
- con.close
276
- raise ex
277
- end
278
- if status['members'].all? { |m| m['health'] == 1 &&
279
- [1, 2, 7].include?(m['state']) } &&
280
- status['members'].any? { |m| m['state'] == 1 }
281
-
282
- connections = []
283
- states = []
284
- status['members'].each do |member|
285
- begin
286
- host, port = member['name'].split(':')
287
- port = port.to_i
288
- conn = Mongo::Connection.new(host, port, :slave_ok => true)
289
- connections << conn
290
- state = conn['admin'].command({:ismaster => 1})
291
- states << state
292
- rescue Mongo::ConnectionFailure
293
- connections.each {|c| c.close }
294
- con.close
295
- raise Mongo::OperationFailure
296
- end
297
- end
298
-
299
- if states.any? {|s| s['ismaster']}
300
- print "all members up!\n\n"
301
- connections.each {|c| c.close }
302
- con.close
303
- return status
304
- else
305
- con.close
306
- raise Mongo::OperationFailure
307
- end
308
- else
309
- con.close
310
- raise Mongo::OperationFailure
311
- end
312
- end
313
- return false
314
- end
315
-
316
- def primary
317
- nodes = get_all_host_pairs_with_state(1)
318
- nodes.empty? ? nil : nodes[0]
319
- end
320
-
321
- def secondaries
322
- get_all_host_pairs_with_state(2)
323
- end
324
-
325
- def arbiters
326
- get_all_host_pairs_with_state(7)
327
- end
328
-
329
- # String used for adding a shard via mongos
330
- # using the addshard command.
331
- def shard_string
332
- str = "#{@name}/"
333
- str << @mongods.map do |k, mongod|
334
- "#{@host}:#{mongod['port']}"
335
- end.join(',')
336
- str
337
- end
338
-
339
- private
340
-
341
- def initiate
342
- puts "Initiating replica set..."
343
- con = get_connection
344
-
345
- attempt do
346
- con.object_id
347
- con['admin'].command({'replSetInitiate' => @config})
348
- end
349
-
350
- con.close
351
- end
352
-
353
- def get_all_nodes_with_state(state)
354
- status = ensure_up
355
- nodes = status['members'].select {|m| m['state'] == state}
356
- nodes = nodes.map do |node|
357
- host_port = node['name'].split(':')
358
- port = host_port[1] ? host_port[1].to_i : 27017
359
- @mongods.keys.detect {|key| @mongods[key]['port'] == port}
360
- end
361
-
362
- nodes == [] ? false : nodes
363
- end
364
-
365
- def get_node_with_state(state)
366
- status = ensure_up
367
- node = status['members'].detect {|m| m['state'] == state}
368
- if node
369
- host_port = node['name'].split(':')
370
- port = host_port[1] ? host_port[1].to_i : 27017
371
- key = @mongods.keys.detect {|n| @mongods[n]['port'] == port}
372
- return key
373
- else
374
- return false
375
- end
376
- end
377
-
378
- def get_all_host_pairs_with_state(state)
379
- status = ensure_up
380
- nodes = status['members'].select {|m| m['state'] == state}
381
- nodes.map do |node|
382
- host_port = node['name'].split(':')
383
- port = host_port[1] ? host_port[1].to_i : 27017
384
- [host, port]
385
- end
386
- end
387
-
388
- def get_connection(node=nil)
389
- con = attempt do
390
- if !node
391
- node = @mongods.keys.detect {|key| !@mongods[key]['arbiterOnly'] && @mongods[key]['up'] }
392
- end
393
- con = Mongo::Connection.new(@host, @mongods[node]['port'], :slave_ok => true)
394
- end
395
-
396
- return con
397
- end
398
-
399
- def get_path(name)
400
- File.join(@path, name)
401
- end
402
-
403
- def attempt(retries=nil)
404
- raise "No block given!" unless block_given?
405
- count = 0
406
-
407
- while count < (retries || @retries) do
408
- begin
409
- return yield
410
- rescue Mongo::OperationFailure, Mongo::ConnectionFailure => ex
411
- sleep(2)
412
- count += 1
413
- end
414
- end
415
-
416
- puts "NO MORE ATTEMPTS"
417
- raise ex
418
- end
419
-
420
- end