fluent-plugin-mongo 0.7.16 → 0.8.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -1
- data/VERSION +1 -1
- data/bin/mongo-tail +37 -30
- data/fluent-plugin-mongo.gemspec +1 -1
- data/lib/fluent/plugin/in_mongo_tail.rb +122 -138
- data/lib/fluent/plugin/logger_support.rb +25 -0
- data/lib/fluent/plugin/mongo_auth.rb +32 -0
- data/lib/fluent/plugin/out_mongo.rb +83 -136
- data/lib/fluent/plugin/out_mongo_replset.rb +20 -31
- data/test/helper.rb +6 -0
- data/test/plugin/test_in_mongo_tail.rb +96 -0
- data/test/plugin/test_out_mongo.rb +282 -0
- data/test/plugin/test_out_mongo_replset.rb +125 -0
- metadata +16 -22
- data/lib/fluent/plugin/mongo_util.rb +0 -27
- data/lib/fluent/plugin/out_mongo_tag_collection.rb +0 -20
- data/test/plugin/in_mongo_tail.rb +0 -73
- data/test/plugin/out_mongo.rb +0 -298
- data/test/plugin/out_mongo_tag_mapped.rb +0 -69
- data/test/test_helper.rb +0 -59
- data/test/tools/auth_repl_set_manager.rb +0 -14
- data/test/tools/repl_set_manager.rb +0 -420
- data/test/tools/rs_test_helper.rb +0 -39
data/test/helper.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
require "helper"
|
2
|
+
|
3
|
+
class MongoTailInputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
setup_mongod
|
7
|
+
end
|
8
|
+
|
9
|
+
def teardown
|
10
|
+
teardown_mongod
|
11
|
+
end
|
12
|
+
|
13
|
+
def collection_name
|
14
|
+
'test'
|
15
|
+
end
|
16
|
+
|
17
|
+
def database_name
|
18
|
+
'fluent_test'
|
19
|
+
end
|
20
|
+
|
21
|
+
def port
|
22
|
+
27017
|
23
|
+
end
|
24
|
+
|
25
|
+
def default_config
|
26
|
+
%[
|
27
|
+
type mongo_tail
|
28
|
+
database test
|
29
|
+
collection log
|
30
|
+
tag_key tag
|
31
|
+
time_key time
|
32
|
+
id_store_file /tmp/fluent_mongo_last_id
|
33
|
+
]
|
34
|
+
end
|
35
|
+
|
36
|
+
def setup_mongod
|
37
|
+
options = {}
|
38
|
+
options[:database] = database_name
|
39
|
+
@client = ::Mongo::Client.new(["localhost:#{port}"], options)
|
40
|
+
end
|
41
|
+
|
42
|
+
def teardown_mongod
|
43
|
+
@client[collection_name].drop
|
44
|
+
end
|
45
|
+
|
46
|
+
def create_driver(conf=default_config)
|
47
|
+
Fluent::Test::InputTestDriver.new(Fluent::MongoTailInput).configure(conf)
|
48
|
+
end
|
49
|
+
|
50
|
+
def test_configure
|
51
|
+
d = create_driver
|
52
|
+
assert_equal('localhost', d.instance.host)
|
53
|
+
assert_equal(27017, d.instance.port)
|
54
|
+
assert_equal('test', d.instance.database)
|
55
|
+
assert_equal('log', d.instance.collection)
|
56
|
+
assert_equal('tag', d.instance.tag_key)
|
57
|
+
assert_equal('time', d.instance.time_key)
|
58
|
+
assert_equal('/tmp/fluent_mongo_last_id', d.instance.id_store_file)
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_configure_with_logger_conf
|
62
|
+
d = create_driver(default_config + %[
|
63
|
+
mongo_log_level error
|
64
|
+
])
|
65
|
+
|
66
|
+
expected = "error"
|
67
|
+
assert_equal(expected, d.instance.mongo_log_level)
|
68
|
+
end
|
69
|
+
|
70
|
+
class MongoAuthenticateTest < self
|
71
|
+
require 'fluent/plugin/mongo_auth'
|
72
|
+
include ::Fluent::MongoAuth
|
73
|
+
|
74
|
+
def setup_mongod
|
75
|
+
options = {}
|
76
|
+
options[:database] = database_name
|
77
|
+
@client = ::Mongo::Client.new(["localhost:#{port}"], options)
|
78
|
+
@client.database.users.create('fluent', password: 'password',
|
79
|
+
roles: [Mongo::Auth::Roles::READ_WRITE])
|
80
|
+
end
|
81
|
+
|
82
|
+
def teardown_mongod
|
83
|
+
@client[collection_name].drop
|
84
|
+
@client.database.users.remove('fluent')
|
85
|
+
end
|
86
|
+
|
87
|
+
def test_authenticate
|
88
|
+
d = create_driver(default_config + %[
|
89
|
+
user fluent
|
90
|
+
password password
|
91
|
+
])
|
92
|
+
|
93
|
+
assert authenticate(@client)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,282 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require "helper"
|
3
|
+
|
4
|
+
class MongoOutputTest < ::Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
Fluent::Test.setup
|
7
|
+
setup_mongod
|
8
|
+
end
|
9
|
+
|
10
|
+
def teardown
|
11
|
+
teardown_mongod
|
12
|
+
end
|
13
|
+
|
14
|
+
def collection_name
|
15
|
+
'test'
|
16
|
+
end
|
17
|
+
|
18
|
+
def database_name
|
19
|
+
'fluent_test'
|
20
|
+
end
|
21
|
+
|
22
|
+
def port
|
23
|
+
27017
|
24
|
+
end
|
25
|
+
|
26
|
+
def default_config
|
27
|
+
%[
|
28
|
+
type mongo
|
29
|
+
database #{database_name}
|
30
|
+
collection #{collection_name}
|
31
|
+
include_time_key true
|
32
|
+
]
|
33
|
+
end
|
34
|
+
|
35
|
+
def setup_mongod
|
36
|
+
options = {}
|
37
|
+
options[:database] = database_name
|
38
|
+
@client = ::Mongo::Client.new(["localhost:#{port}"], options)
|
39
|
+
end
|
40
|
+
|
41
|
+
def teardown_mongod
|
42
|
+
@client[collection_name].drop
|
43
|
+
end
|
44
|
+
|
45
|
+
def create_driver(conf=default_config, tag='test')
|
46
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput, tag).configure(conf)
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_configure
|
50
|
+
d = create_driver(%[
|
51
|
+
type mongo
|
52
|
+
database fluent_test
|
53
|
+
collection test_collection
|
54
|
+
|
55
|
+
capped
|
56
|
+
capped_size 100
|
57
|
+
])
|
58
|
+
|
59
|
+
assert_equal('fluent_test', d.instance.database)
|
60
|
+
assert_equal('test_collection', d.instance.collection)
|
61
|
+
assert_equal('localhost', d.instance.host)
|
62
|
+
assert_equal(port, d.instance.port)
|
63
|
+
assert_equal({capped: true, size: 100}, d.instance.collection_options)
|
64
|
+
assert_equal({ssl: false, write: {j: false}}, d.instance.client_options)
|
65
|
+
end
|
66
|
+
|
67
|
+
def test_configure_with_ssl
|
68
|
+
conf = default_config + %[
|
69
|
+
ssl true
|
70
|
+
]
|
71
|
+
d = create_driver(conf)
|
72
|
+
expected = {
|
73
|
+
write: {
|
74
|
+
j: false,
|
75
|
+
},
|
76
|
+
ssl: true,
|
77
|
+
ssl_cert: nil,
|
78
|
+
ssl_key: nil,
|
79
|
+
ssl_key_pass_phrase: nil,
|
80
|
+
ssl_verify: false,
|
81
|
+
ssl_ca_cert: nil,
|
82
|
+
}
|
83
|
+
assert_equal(expected, d.instance.client_options)
|
84
|
+
end
|
85
|
+
|
86
|
+
def test_configure_with_tag_mapped
|
87
|
+
conf = default_config + %[
|
88
|
+
tag_mapped true
|
89
|
+
remove_tag_prefix raw.
|
90
|
+
]
|
91
|
+
d = create_driver(conf)
|
92
|
+
assert_true(d.instance.tag_mapped)
|
93
|
+
assert_equal(/^raw\./, d.instance.remove_tag_prefix)
|
94
|
+
end
|
95
|
+
|
96
|
+
def test_configure_with_write_concern
|
97
|
+
d = create_driver(default_config + %[
|
98
|
+
write_concern 2
|
99
|
+
])
|
100
|
+
|
101
|
+
expected = {
|
102
|
+
ssl: false,
|
103
|
+
write: {
|
104
|
+
j: false,
|
105
|
+
w: 2,
|
106
|
+
},
|
107
|
+
}
|
108
|
+
assert_equal(expected, d.instance.client_options)
|
109
|
+
end
|
110
|
+
|
111
|
+
def test_configure_with_journaled
|
112
|
+
d = create_driver(default_config + %[
|
113
|
+
journaled true
|
114
|
+
])
|
115
|
+
|
116
|
+
expected = {
|
117
|
+
ssl: false,
|
118
|
+
write: {
|
119
|
+
j: true,
|
120
|
+
},
|
121
|
+
}
|
122
|
+
assert_equal(expected, d.instance.client_options)
|
123
|
+
end
|
124
|
+
|
125
|
+
def test_configure_with_logger_conf
|
126
|
+
d = create_driver(default_config + %[
|
127
|
+
mongo_log_level fatal
|
128
|
+
])
|
129
|
+
|
130
|
+
expected = "fatal"
|
131
|
+
assert_equal(expected, d.instance.mongo_log_level)
|
132
|
+
end
|
133
|
+
|
134
|
+
def get_documents
|
135
|
+
@client[collection_name].find.to_a.map {|e| e.delete('_id'); e}
|
136
|
+
end
|
137
|
+
|
138
|
+
def emit_documents(d)
|
139
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
140
|
+
d.emit({'a' => 1}, time)
|
141
|
+
d.emit({'a' => 2}, time)
|
142
|
+
time
|
143
|
+
end
|
144
|
+
|
145
|
+
def test_format
|
146
|
+
d = create_driver
|
147
|
+
|
148
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
149
|
+
d.emit({'a' => 1}, time)
|
150
|
+
d.emit({'a' => 2}, time)
|
151
|
+
d.expect_format([time, {'a' => 1, d.instance.time_key => time}].to_msgpack)
|
152
|
+
d.expect_format([time, {'a' => 2, d.instance.time_key => time}].to_msgpack)
|
153
|
+
d.run
|
154
|
+
|
155
|
+
documents = get_documents
|
156
|
+
assert_equal(2, documents.size)
|
157
|
+
end
|
158
|
+
|
159
|
+
def test_write
|
160
|
+
d = create_driver
|
161
|
+
t = emit_documents(d)
|
162
|
+
|
163
|
+
d.run
|
164
|
+
actual_documents = get_documents
|
165
|
+
time = Time.parse("2011-01-02 13:14:15 UTC")
|
166
|
+
expected = [{'a' => 1, d.instance.time_key => time},
|
167
|
+
{'a' => 2, d.instance.time_key => time}]
|
168
|
+
assert_equal(expected, actual_documents)
|
169
|
+
end
|
170
|
+
|
171
|
+
def test_write_at_enable_tag
|
172
|
+
d = create_driver(default_config + %[
|
173
|
+
include_tag_key true
|
174
|
+
include_time_key false
|
175
|
+
])
|
176
|
+
t = emit_documents(d)
|
177
|
+
|
178
|
+
d.run
|
179
|
+
actual_documents = get_documents
|
180
|
+
expected = [{'a' => 1, d.instance.tag_key => 'test'},
|
181
|
+
{'a' => 2, d.instance.tag_key => 'test'}]
|
182
|
+
assert_equal(expected, actual_documents)
|
183
|
+
end
|
184
|
+
|
185
|
+
def emit_invalid_documents(d)
|
186
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
187
|
+
d.emit({'a' => 3, 'b' => "c", '$last' => '石動'}, time)
|
188
|
+
d.emit({'a' => 4, 'b' => "d", 'first' => '菖蒲'.encode('EUC-JP').force_encoding('UTF-8')}, time)
|
189
|
+
time
|
190
|
+
end
|
191
|
+
|
192
|
+
def test_write_with_invalid_recoreds_with_keys_containing_dot_and_dollar
|
193
|
+
d = create_driver(default_config + %[
|
194
|
+
replace_dot_in_key_with _dot_
|
195
|
+
replace_dollar_in_key_with _dollar_
|
196
|
+
])
|
197
|
+
|
198
|
+
original_time = "2016-02-01 13:14:15 UTC"
|
199
|
+
time = Time.parse(original_time).to_i
|
200
|
+
d.emit({
|
201
|
+
"foo.bar1" => {
|
202
|
+
"$foo$bar" => "baz"
|
203
|
+
},
|
204
|
+
"foo.bar2" => [
|
205
|
+
{
|
206
|
+
"$foo$bar" => "baz"
|
207
|
+
}
|
208
|
+
],
|
209
|
+
}, time)
|
210
|
+
d.run
|
211
|
+
|
212
|
+
documents = get_documents
|
213
|
+
expected = {"foo_dot_bar1" => {
|
214
|
+
"_dollar_foo$bar"=>"baz"
|
215
|
+
},
|
216
|
+
"foo_dot_bar2" => [
|
217
|
+
{
|
218
|
+
"_dollar_foo$bar"=>"baz"
|
219
|
+
},
|
220
|
+
], "time" => Time.parse(original_time)
|
221
|
+
}
|
222
|
+
assert_equal(1, documents.size)
|
223
|
+
assert_equal(expected, documents[0])
|
224
|
+
end
|
225
|
+
|
226
|
+
class WithAuthenticateTest < self
|
227
|
+
def setup_mongod
|
228
|
+
options = {}
|
229
|
+
options[:database] = database_name
|
230
|
+
@client = ::Mongo::Client.new(["localhost:#{port}"], options)
|
231
|
+
@client.database.users.create('fluent', password: 'password',
|
232
|
+
roles: [Mongo::Auth::Roles::READ_WRITE])
|
233
|
+
end
|
234
|
+
|
235
|
+
def teardown_mongod
|
236
|
+
@client[collection_name].drop
|
237
|
+
@client.database.users.remove('fluent')
|
238
|
+
end
|
239
|
+
|
240
|
+
def test_write_with_authenticate
|
241
|
+
d = create_driver(default_config + %[
|
242
|
+
user fluent
|
243
|
+
password password
|
244
|
+
])
|
245
|
+
t = emit_documents(d)
|
246
|
+
|
247
|
+
d.run
|
248
|
+
actual_documents = get_documents
|
249
|
+
time = Time.parse("2011-01-02 13:14:15 UTC")
|
250
|
+
expected = [{'a' => 1, d.instance.time_key => time},
|
251
|
+
{'a' => 2, d.instance.time_key => time}]
|
252
|
+
assert_equal(expected, actual_documents)
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
class MongoAuthenticateTest < self
|
257
|
+
require 'fluent/plugin/mongo_auth'
|
258
|
+
include ::Fluent::MongoAuth
|
259
|
+
|
260
|
+
def setup_mongod
|
261
|
+
options = {}
|
262
|
+
options[:database] = database_name
|
263
|
+
@client = ::Mongo::Client.new(["localhost:#{port}"], options)
|
264
|
+
@client.database.users.create('fluent', password: 'password',
|
265
|
+
roles: [Mongo::Auth::Roles::READ_WRITE])
|
266
|
+
end
|
267
|
+
|
268
|
+
def teardown_mongod
|
269
|
+
@client[collection_name].drop
|
270
|
+
@client.database.users.remove('fluent')
|
271
|
+
end
|
272
|
+
|
273
|
+
def test_authenticate
|
274
|
+
d = create_driver(default_config + %[
|
275
|
+
user fluent
|
276
|
+
password password
|
277
|
+
])
|
278
|
+
|
279
|
+
assert authenticate(@client)
|
280
|
+
end
|
281
|
+
end
|
282
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
require "helper"
|
2
|
+
|
3
|
+
class MongoReplsetOutputTest < ::Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
def teardown
|
9
|
+
end
|
10
|
+
|
11
|
+
def collection_name
|
12
|
+
'test'
|
13
|
+
end
|
14
|
+
|
15
|
+
def database_name
|
16
|
+
'fluent_test'
|
17
|
+
end
|
18
|
+
|
19
|
+
def port
|
20
|
+
27018
|
21
|
+
end
|
22
|
+
|
23
|
+
def default_config
|
24
|
+
%[
|
25
|
+
type mongo
|
26
|
+
port 27018
|
27
|
+
database #{database_name}
|
28
|
+
collection #{collection_name}
|
29
|
+
include_time_key true
|
30
|
+
replica_set rs0
|
31
|
+
]
|
32
|
+
end
|
33
|
+
|
34
|
+
def create_driver(conf=default_config, tag='test')
|
35
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputReplset, tag).configure(conf)
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_configure
|
39
|
+
d = create_driver(%[
|
40
|
+
type mongo
|
41
|
+
port 27018
|
42
|
+
database fluent_test
|
43
|
+
collection test_collection
|
44
|
+
|
45
|
+
replica_set rs0
|
46
|
+
])
|
47
|
+
|
48
|
+
assert_equal('fluent_test', d.instance.database)
|
49
|
+
assert_equal('test_collection', d.instance.collection)
|
50
|
+
assert_equal('localhost', d.instance.host)
|
51
|
+
assert_equal(port, d.instance.port)
|
52
|
+
assert_equal({replica_set: 'rs0', :ssl=>false, :write=>{:j=>false}},
|
53
|
+
d.instance.client_options)
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_configure_with_logger_conf
|
57
|
+
d = create_driver(default_config + %[
|
58
|
+
mongo_log_level fatal
|
59
|
+
])
|
60
|
+
|
61
|
+
expected = "fatal"
|
62
|
+
assert_equal(expected, d.instance.mongo_log_level)
|
63
|
+
end
|
64
|
+
|
65
|
+
class ReplisetWriteTest < self
|
66
|
+
def setup
|
67
|
+
omit("Replica set setup is too hard in CI.") if ENV['CI']
|
68
|
+
|
69
|
+
setup_mongod
|
70
|
+
end
|
71
|
+
|
72
|
+
def teardown
|
73
|
+
omit("Replica set setup is too hard in CI.") if ENV['CI']
|
74
|
+
|
75
|
+
teardown_mongod
|
76
|
+
end
|
77
|
+
|
78
|
+
def setup_mongod
|
79
|
+
options = {}
|
80
|
+
options[:database] = database_name
|
81
|
+
@client = ::Mongo::Client.new(["localhost:#{port}"], options)
|
82
|
+
end
|
83
|
+
|
84
|
+
def teardown_mongod
|
85
|
+
@client[collection_name].drop
|
86
|
+
end
|
87
|
+
|
88
|
+
def get_documents
|
89
|
+
@client[collection_name].find.to_a.map {|e| e.delete('_id'); e}
|
90
|
+
end
|
91
|
+
|
92
|
+
def emit_documents(d)
|
93
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
94
|
+
d.emit({'a' => 1}, time)
|
95
|
+
d.emit({'a' => 2}, time)
|
96
|
+
time
|
97
|
+
end
|
98
|
+
|
99
|
+
def test_format
|
100
|
+
d = create_driver
|
101
|
+
|
102
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
103
|
+
d.emit({'a' => 1}, time)
|
104
|
+
d.emit({'a' => 2}, time)
|
105
|
+
d.expect_format([time, {'a' => 1, d.instance.time_key => time}].to_msgpack)
|
106
|
+
d.expect_format([time, {'a' => 2, d.instance.time_key => time}].to_msgpack)
|
107
|
+
d.run
|
108
|
+
|
109
|
+
documents = get_documents
|
110
|
+
assert_equal(2, documents.size)
|
111
|
+
end
|
112
|
+
|
113
|
+
def test_write
|
114
|
+
d = create_driver
|
115
|
+
t = emit_documents(d)
|
116
|
+
|
117
|
+
d.run
|
118
|
+
actual_documents = get_documents
|
119
|
+
time = Time.parse("2011-01-02 13:14:15 UTC")
|
120
|
+
expected = [{'a' => 1, d.instance.time_key => time},
|
121
|
+
{'a' => 2, d.instance.time_key => time}]
|
122
|
+
assert_equal(expected, actual_documents)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|