fluent-plugin-mongo 0.7.12 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +5 -0
- data/Rakefile +1 -1
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_mongo_tail.rb +2 -0
- data/lib/fluent/plugin/out_mongo.rb +29 -42
- data/test/plugin/in_mongo_tail.rb +1 -1
- data/test/plugin/out_mongo.rb +77 -58
- data/test/plugin/out_mongo_tag_mapped.rb +1 -1
- data/test/test_helper.rb +15 -21
- data/test/tools/repl_set_manager.rb +10 -5
- data/test/tools/rs_test_helper.rb +8 -8
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c00835bf3c3c44d2055e59c9d38d8654c438f83e
|
4
|
+
data.tar.gz: 21f1d514533a48beddf1391b68a2467e6131c73c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 83fcb6008050f5eb175764dfaf1ab96836c41168947cd24ced230257fe60d17d04d7de60acfc9c18fe08eda59b103ad39b15b5b1ccbb76ac6f54d2e5c02f04ac
|
7
|
+
data.tar.gz: ca4144e0884ee794f8cdd1ef68f65a5e83748bd37b45a1c5a037f9a03cbad9f828292ab8200ab02e70dc3a41e66471e57a646c7855fd8cf7572f25dd97596eb9
|
data/ChangeLog
CHANGED
data/Rakefile
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.7.
|
1
|
+
0.7.13
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'fluent/output'
|
2
|
+
|
1
3
|
module Fluent
|
2
4
|
class MongoOutput < BufferedOutput
|
3
5
|
Plugin.register_output('mongo', self)
|
@@ -36,6 +38,9 @@ module Fluent
|
|
36
38
|
config_param :ssl_verify, :bool, :default => false
|
37
39
|
config_param :ssl_ca_cert, :string, :default => nil
|
38
40
|
|
41
|
+
# For older (1.7 or earlier) MongoDB versions
|
42
|
+
config_param :mongodb_smaller_bson_limit, :bool, :default => false
|
43
|
+
|
39
44
|
attr_reader :collection_options, :connection_options
|
40
45
|
|
41
46
|
unless method_defined?(:log)
|
@@ -52,7 +57,31 @@ module Fluent
|
|
52
57
|
@collection_options = {:capped => false}
|
53
58
|
end
|
54
59
|
|
60
|
+
# Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
|
61
|
+
LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024 # 2MB = 4MB / 2
|
62
|
+
LIMIT_AFTER_v1_8 = 8 * 1024 * 1024 # 8MB = 16MB / 2
|
63
|
+
|
55
64
|
def configure(conf)
|
65
|
+
if conf.has_key?('buffer_chunk_limit')
|
66
|
+
configured_chunk_limit_size = Config.size_value(conf['buffer_chunk_limit'])
|
67
|
+
estimated_limit_size = LIMIT_AFTER_v1_8
|
68
|
+
estimated_limit_size_conf = '8m'
|
69
|
+
if conf.has_key?('mongodb_smaller_bson_limit') && Config.bool_value(conf['mongodb_smaller_bson_limit'])
|
70
|
+
estimated_limit_size = LIMIT_BEFORE_v1_8
|
71
|
+
estimated_limit_size_conf = '2m'
|
72
|
+
end
|
73
|
+
if configured_chunk_limit_size > estimated_limit_size
|
74
|
+
log.warn ":buffer_chunk_limit(#{conf['buffer_chunk_limit']}) is large. Reset :buffer_chunk_limit with #{estimated_limit_size_conf}"
|
75
|
+
conf['buffer_chunk_limit'] = estimated_limit_size_conf
|
76
|
+
end
|
77
|
+
else
|
78
|
+
if conf.has_key?('mongodb_smaller_bson_limit') && Config.bool_value(conf['mongodb_smaller_bson_limit'])
|
79
|
+
conf['buffer_chunk_limit'] = '2m'
|
80
|
+
else
|
81
|
+
conf['buffer_chunk_limit'] = '8m'
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
56
85
|
super
|
57
86
|
|
58
87
|
if conf.has_key?('tag_mapped')
|
@@ -102,9 +131,6 @@ module Fluent
|
|
102
131
|
# Non tag mapped mode, we can check collection configuration before server start.
|
103
132
|
get_or_create_collection(@collection) unless @tag_mapped
|
104
133
|
|
105
|
-
# From configure for avoding complex method dependency...
|
106
|
-
@buffer.buffer_chunk_limit = available_buffer_chunk_limit
|
107
|
-
|
108
134
|
super
|
109
135
|
end
|
110
136
|
|
@@ -232,45 +258,6 @@ module Fluent
|
|
232
258
|
authenticate(db)
|
233
259
|
end
|
234
260
|
|
235
|
-
# Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
|
236
|
-
LIMIT_BEFORE_v1_8 = 2 * 1024 * 1024 # 2MB = 4MB / 2
|
237
|
-
LIMIT_AFTER_v1_8 = 8 * 1024 * 1024 # 8MB = 16MB / 2
|
238
|
-
|
239
|
-
def available_buffer_chunk_limit
|
240
|
-
begin
|
241
|
-
limit = mongod_version >= "1.8.0" ? LIMIT_AFTER_v1_8 : LIMIT_BEFORE_v1_8
|
242
|
-
rescue Mongo::ConnectionFailure => e
|
243
|
-
log.fatal "Failed to connect to 'mongod'. Please restart 'fluentd' after 'mongod' started: #{e}"
|
244
|
-
exit!
|
245
|
-
rescue Mongo::OperationFailure => e
|
246
|
-
log.fatal "Operation failed. Probably, 'mongod' needs an authentication: #{e}"
|
247
|
-
exit!
|
248
|
-
rescue Exception => e
|
249
|
-
log.warn "mongo unknown error #{e}, set #{LIMIT_BEFORE_v1_8} to chunk limit"
|
250
|
-
limit = LIMIT_BEFORE_v1_8
|
251
|
-
end
|
252
|
-
|
253
|
-
if @buffer.buffer_chunk_limit > limit
|
254
|
-
log.warn ":buffer_chunk_limit(#{@buffer.buffer_chunk_limit}) is large. Reset :buffer_chunk_limit with #{limit}"
|
255
|
-
limit
|
256
|
-
else
|
257
|
-
@buffer.buffer_chunk_limit
|
258
|
-
end
|
259
|
-
end
|
260
|
-
|
261
|
-
def mongod_version
|
262
|
-
version = nil
|
263
|
-
|
264
|
-
begin
|
265
|
-
version = get_connection.command('buildInfo' => 1)['version']
|
266
|
-
rescue Mongo::OperationFailure
|
267
|
-
# fallback for buggy mongod version support
|
268
|
-
version = authenticate(Mongo::MongoClient.new(@host, @port, @connection_options).db('admin')).command('buildInfo' => 1)['version']
|
269
|
-
end
|
270
|
-
|
271
|
-
version
|
272
|
-
end
|
273
|
-
|
274
261
|
def replace_key_of_hash(hash_or_array, pattern, replacement)
|
275
262
|
case hash_or_array
|
276
263
|
when Array
|
data/test/plugin/out_mongo.rb
CHANGED
@@ -1,63 +1,12 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
require 'tools/rs_test_helper'
|
3
|
+
require 'fluent/plugin/out_mongo'
|
3
4
|
|
4
|
-
|
5
|
-
include MongoTestHelper
|
6
|
-
|
7
|
-
def setup
|
8
|
-
Fluent::Test.setup
|
9
|
-
require 'fluent/plugin/out_mongo'
|
10
|
-
|
11
|
-
setup_mongod
|
12
|
-
end
|
13
|
-
|
14
|
-
def teardown
|
15
|
-
@db.collection(collection_name).drop
|
16
|
-
teardown_mongod
|
17
|
-
end
|
18
|
-
|
5
|
+
module MongoOutputTestCases
|
19
6
|
def collection_name
|
20
7
|
'test'
|
21
8
|
end
|
22
9
|
|
23
|
-
def default_config
|
24
|
-
%[
|
25
|
-
type mongo
|
26
|
-
database #{MONGO_DB_DB}
|
27
|
-
collection #{collection_name}
|
28
|
-
include_time_key true # TestDriver ignore config_set_default?
|
29
|
-
]
|
30
|
-
end
|
31
|
-
|
32
|
-
def create_driver(conf = default_config)
|
33
|
-
conf = conf + %[
|
34
|
-
port #{@@mongod_port}
|
35
|
-
]
|
36
|
-
@db = Mongo::MongoClient.new('localhost', @@mongod_port).db(MONGO_DB_DB)
|
37
|
-
Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput).configure(conf)
|
38
|
-
end
|
39
|
-
|
40
|
-
def test_configure
|
41
|
-
d = create_driver(%[
|
42
|
-
type mongo
|
43
|
-
database fluent_test
|
44
|
-
collection test_collection
|
45
|
-
|
46
|
-
capped
|
47
|
-
capped_size 100
|
48
|
-
])
|
49
|
-
|
50
|
-
assert_equal('fluent_test', d.instance.database)
|
51
|
-
assert_equal('test_collection', d.instance.collection)
|
52
|
-
assert_equal('localhost', d.instance.host)
|
53
|
-
assert_equal(@@mongod_port, d.instance.port)
|
54
|
-
assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
|
55
|
-
assert_equal({:ssl => false, :pool_size => 1, :j => false}, d.instance.connection_options)
|
56
|
-
# buffer_chunk_limit moved from configure to start
|
57
|
-
# I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
|
58
|
-
# assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
|
59
|
-
end
|
60
|
-
|
61
10
|
def test_configure_with_write_concern
|
62
11
|
d = create_driver(default_config + %[
|
63
12
|
write_concern 2
|
@@ -223,16 +172,86 @@ class MongoOutputTest < Test::Unit::TestCase
|
|
223
172
|
end
|
224
173
|
end
|
225
174
|
|
226
|
-
class
|
175
|
+
class MongoOutputTest < Test::Unit::TestCase
|
176
|
+
include MongoOutputTestCases
|
177
|
+
|
178
|
+
class << self
|
179
|
+
def startup
|
180
|
+
MongoTestHelper.setup_mongod
|
181
|
+
end
|
182
|
+
|
183
|
+
def shutdown
|
184
|
+
MongoTestHelper.teardown_mongod
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
def setup
|
189
|
+
Fluent::Test.setup
|
190
|
+
end
|
191
|
+
|
192
|
+
def teardown
|
193
|
+
@db.collection(collection_name).drop
|
194
|
+
end
|
195
|
+
|
196
|
+
def default_config
|
197
|
+
%[
|
198
|
+
type mongo
|
199
|
+
database #{MONGO_DB_DB}
|
200
|
+
collection #{collection_name}
|
201
|
+
include_time_key true # TestDriver ignore config_set_default?
|
202
|
+
]
|
203
|
+
end
|
204
|
+
|
205
|
+
def create_driver(conf = default_config)
|
206
|
+
conf = conf + %[
|
207
|
+
port #{MongoTestHelper.mongod_port}
|
208
|
+
]
|
209
|
+
@db = Mongo::MongoClient.new('localhost', MongoTestHelper.mongod_port).db(MONGO_DB_DB)
|
210
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput).configure(conf)
|
211
|
+
end
|
212
|
+
|
213
|
+
def test_configure
|
214
|
+
d = create_driver(%[
|
215
|
+
type mongo
|
216
|
+
database fluent_test
|
217
|
+
collection test_collection
|
218
|
+
|
219
|
+
capped
|
220
|
+
capped_size 100
|
221
|
+
])
|
222
|
+
|
223
|
+
assert_equal('fluent_test', d.instance.database)
|
224
|
+
assert_equal('test_collection', d.instance.collection)
|
225
|
+
assert_equal('localhost', d.instance.host)
|
226
|
+
assert_equal(MongoTestHelper.mongod_port, d.instance.port)
|
227
|
+
assert_equal({:capped => true, :size => 100}, d.instance.collection_options)
|
228
|
+
assert_equal({:ssl => false, :pool_size => 1, :j => false}, d.instance.connection_options)
|
229
|
+
# buffer_chunk_limit moved from configure to start
|
230
|
+
# I will move this test to correct space after BufferedOutputTestDriver supports start method invoking
|
231
|
+
# assert_equal(Fluent::MongoOutput::LIMIT_BEFORE_v1_8, d.instance.instance_variable_get(:@buffer).buffer_chunk_limit)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
class MongoReplOutputTest < Test::Unit::TestCase
|
236
|
+
include MongoOutputTestCases
|
237
|
+
|
238
|
+
class << self
|
239
|
+
def startup
|
240
|
+
setup_rs
|
241
|
+
end
|
242
|
+
|
243
|
+
def shutdown
|
244
|
+
teardown_rs
|
245
|
+
end
|
246
|
+
end
|
247
|
+
|
227
248
|
def setup
|
228
249
|
Fluent::Test.setup
|
229
250
|
require 'fluent/plugin/out_mongo_replset'
|
230
|
-
|
231
|
-
ensure_rs
|
232
251
|
end
|
233
252
|
|
234
253
|
def teardown
|
235
|
-
|
254
|
+
@@rs.restart_killed_nodes
|
236
255
|
if defined?(@db) && @db
|
237
256
|
@db.collection(collection_name).drop
|
238
257
|
@db.connection.close
|
@@ -251,7 +270,7 @@ class MongoReplOutputTest < MongoOutputTest
|
|
251
270
|
end
|
252
271
|
|
253
272
|
def create_driver(conf = default_config)
|
254
|
-
@db = Mongo::MongoReplicaSetClient.new(build_seeds(3), :name =>
|
273
|
+
@db = Mongo::MongoReplicaSetClient.new(build_seeds(3), :name => @@rs.name).db(MONGO_DB_DB)
|
255
274
|
Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputReplset).configure(conf)
|
256
275
|
end
|
257
276
|
|
data/test/test_helper.rb
CHANGED
@@ -33,33 +33,27 @@ MONGO_DB_DB = 'fluent_test'
|
|
33
33
|
MONGO_DB_PATH = File.join(File.dirname(__FILE__), 'plugin', 'data')
|
34
34
|
|
35
35
|
module MongoTestHelper
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
system("killall mongod")
|
36
|
+
def self.cleanup_mongod_env
|
37
|
+
Process.kill "TERM", @@pid
|
38
|
+
Process.waitpid @@pid
|
40
39
|
system("rm -rf #{MONGO_DB_PATH}")
|
41
|
-
system("mkdir -p #{MONGO_DB_PATH}")
|
42
40
|
end
|
43
41
|
|
44
|
-
def setup_mongod
|
45
|
-
|
46
|
-
|
42
|
+
def self.setup_mongod
|
43
|
+
system("rm -rf #{MONGO_DB_PATH}")
|
44
|
+
system("mkdir -p #{MONGO_DB_PATH}")
|
47
45
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
end
|
46
|
+
@@mongod_port = unused_port
|
47
|
+
@@pid = spawn(ENV['mongod'], "--port=#{@@mongod_port}", "--dbpath=#{MONGO_DB_PATH}")
|
48
|
+
sleep 3
|
49
|
+
end
|
53
50
|
|
54
|
-
|
51
|
+
def self.teardown_mongod
|
52
|
+
Mongo::Connection.new('localhost', @@mongod_port).drop_database(MONGO_DB_DB)
|
53
|
+
cleanup_mongod_env
|
55
54
|
end
|
56
55
|
|
57
|
-
def
|
58
|
-
|
59
|
-
Mongo::Connection.new('localhost', @@mongod_port).drop_database(MONGO_DB_DB)
|
60
|
-
end
|
61
|
-
if @@setup_count == self.class.methods.size
|
62
|
-
cleanup_mongod_env
|
63
|
-
end
|
56
|
+
def self.mongod_port
|
57
|
+
@@mongod_port
|
64
58
|
end
|
65
59
|
end
|
@@ -45,8 +45,9 @@ class ReplSetManager
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def start_set
|
48
|
-
|
49
|
-
|
48
|
+
@mongods.keys.each do |node|
|
49
|
+
kill(node)
|
50
|
+
end
|
50
51
|
should_start = true
|
51
52
|
puts "** Starting a replica set with #{@count} nodes"
|
52
53
|
|
@@ -79,7 +80,9 @@ class ReplSetManager
|
|
79
80
|
end
|
80
81
|
|
81
82
|
def cleanup_set
|
82
|
-
|
83
|
+
@mongods.keys.each do |node|
|
84
|
+
kill(node)
|
85
|
+
end
|
83
86
|
@count.times do |n|
|
84
87
|
system("rm -rf #{@mongods[n]['db_path']}")
|
85
88
|
end
|
@@ -193,9 +196,11 @@ class ReplSetManager
|
|
193
196
|
end
|
194
197
|
|
195
198
|
def kill(node, signal=2)
|
199
|
+
return unless @mongods[node]['up']
|
200
|
+
|
196
201
|
pid = @mongods[node]['pid']
|
197
202
|
puts "** Killing node with pid #{pid} at port #{@mongods[node]['port']}"
|
198
|
-
|
203
|
+
Process.kill signal, pid
|
199
204
|
dead = wait_for_death(pid)
|
200
205
|
@mongods[node]['up'] = false if dead
|
201
206
|
end
|
@@ -254,7 +259,7 @@ class ReplSetManager
|
|
254
259
|
system(@mongods[node]['start'])
|
255
260
|
@mongods[node]['up'] = true
|
256
261
|
sleep(0.5)
|
257
|
-
@mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip
|
262
|
+
@mongods[node]['pid'] = File.open(File.join(@mongods[node]['db_path'], 'mongod.lock')).read.strip.to_i
|
258
263
|
end
|
259
264
|
alias :restart :start
|
260
265
|
|
@@ -5,13 +5,13 @@ require 'tools/repl_set_manager'
|
|
5
5
|
class Test::Unit::TestCase
|
6
6
|
# Ensure replica set is available as an instance variable and that
|
7
7
|
# a new set is spun up for each TestCase class
|
8
|
-
def
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
8
|
+
def self.setup_rs
|
9
|
+
@@rs = ReplSetManager.new
|
10
|
+
@@rs.start_set
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.teardown_rs
|
14
|
+
@@rs.cleanup_set
|
15
15
|
end
|
16
16
|
|
17
17
|
# Generic code for rescuing connection failures and retrying operations.
|
@@ -32,7 +32,7 @@ class Test::Unit::TestCase
|
|
32
32
|
def build_seeds(num_hosts)
|
33
33
|
seeds = []
|
34
34
|
num_hosts.times do |n|
|
35
|
-
seeds << "#{
|
35
|
+
seeds << "#{@@rs.host}:#{@@rs.ports[n]}"
|
36
36
|
end
|
37
37
|
seeds
|
38
38
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mongo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-06-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -149,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
149
|
version: '0'
|
150
150
|
requirements: []
|
151
151
|
rubyforge_project:
|
152
|
-
rubygems_version: 2.
|
152
|
+
rubygems_version: 2.5.1
|
153
153
|
signing_key:
|
154
154
|
specification_version: 4
|
155
155
|
summary: MongoDB plugin for Fluentd
|