fluent-plugin-mongo 0.7.16 → 0.8.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/test/helper.rb ADDED
@@ -0,0 +1,6 @@
1
+ require 'test/unit'
2
+ require 'fluent/test'
3
+ require 'mongo'
4
+ require 'fluent/plugin/out_mongo'
5
+ require 'fluent/plugin/out_mongo_replset'
6
+ require 'fluent/plugin/in_mongo_tail'
@@ -0,0 +1,96 @@
1
+ require "helper"
2
+
3
+ class MongoTailInputTest < Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ setup_mongod
7
+ end
8
+
9
+ def teardown
10
+ teardown_mongod
11
+ end
12
+
13
+ def collection_name
14
+ 'test'
15
+ end
16
+
17
+ def database_name
18
+ 'fluent_test'
19
+ end
20
+
21
+ def port
22
+ 27017
23
+ end
24
+
25
+ def default_config
26
+ %[
27
+ type mongo_tail
28
+ database test
29
+ collection log
30
+ tag_key tag
31
+ time_key time
32
+ id_store_file /tmp/fluent_mongo_last_id
33
+ ]
34
+ end
35
+
36
+ def setup_mongod
37
+ options = {}
38
+ options[:database] = database_name
39
+ @client = ::Mongo::Client.new(["localhost:#{port}"], options)
40
+ end
41
+
42
+ def teardown_mongod
43
+ @client[collection_name].drop
44
+ end
45
+
46
+ def create_driver(conf=default_config)
47
+ Fluent::Test::InputTestDriver.new(Fluent::MongoTailInput).configure(conf)
48
+ end
49
+
50
+ def test_configure
51
+ d = create_driver
52
+ assert_equal('localhost', d.instance.host)
53
+ assert_equal(27017, d.instance.port)
54
+ assert_equal('test', d.instance.database)
55
+ assert_equal('log', d.instance.collection)
56
+ assert_equal('tag', d.instance.tag_key)
57
+ assert_equal('time', d.instance.time_key)
58
+ assert_equal('/tmp/fluent_mongo_last_id', d.instance.id_store_file)
59
+ end
60
+
61
+ def test_configure_with_logger_conf
62
+ d = create_driver(default_config + %[
63
+ mongo_log_level error
64
+ ])
65
+
66
+ expected = "error"
67
+ assert_equal(expected, d.instance.mongo_log_level)
68
+ end
69
+
70
+ class MongoAuthenticateTest < self
71
+ require 'fluent/plugin/mongo_auth'
72
+ include ::Fluent::MongoAuth
73
+
74
+ def setup_mongod
75
+ options = {}
76
+ options[:database] = database_name
77
+ @client = ::Mongo::Client.new(["localhost:#{port}"], options)
78
+ @client.database.users.create('fluent', password: 'password',
79
+ roles: [Mongo::Auth::Roles::READ_WRITE])
80
+ end
81
+
82
+ def teardown_mongod
83
+ @client[collection_name].drop
84
+ @client.database.users.remove('fluent')
85
+ end
86
+
87
+ def test_authenticate
88
+ d = create_driver(default_config + %[
89
+ user fluent
90
+ password password
91
+ ])
92
+
93
+ assert authenticate(@client)
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,282 @@
1
+ # coding: utf-8
2
+ require "helper"
3
+
4
+ class MongoOutputTest < ::Test::Unit::TestCase
5
+ def setup
6
+ Fluent::Test.setup
7
+ setup_mongod
8
+ end
9
+
10
+ def teardown
11
+ teardown_mongod
12
+ end
13
+
14
+ def collection_name
15
+ 'test'
16
+ end
17
+
18
+ def database_name
19
+ 'fluent_test'
20
+ end
21
+
22
+ def port
23
+ 27017
24
+ end
25
+
26
+ def default_config
27
+ %[
28
+ type mongo
29
+ database #{database_name}
30
+ collection #{collection_name}
31
+ include_time_key true
32
+ ]
33
+ end
34
+
35
+ def setup_mongod
36
+ options = {}
37
+ options[:database] = database_name
38
+ @client = ::Mongo::Client.new(["localhost:#{port}"], options)
39
+ end
40
+
41
+ def teardown_mongod
42
+ @client[collection_name].drop
43
+ end
44
+
45
+ def create_driver(conf=default_config, tag='test')
46
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutput, tag).configure(conf)
47
+ end
48
+
49
+ def test_configure
50
+ d = create_driver(%[
51
+ type mongo
52
+ database fluent_test
53
+ collection test_collection
54
+
55
+ capped
56
+ capped_size 100
57
+ ])
58
+
59
+ assert_equal('fluent_test', d.instance.database)
60
+ assert_equal('test_collection', d.instance.collection)
61
+ assert_equal('localhost', d.instance.host)
62
+ assert_equal(port, d.instance.port)
63
+ assert_equal({capped: true, size: 100}, d.instance.collection_options)
64
+ assert_equal({ssl: false, write: {j: false}}, d.instance.client_options)
65
+ end
66
+
67
+ def test_configure_with_ssl
68
+ conf = default_config + %[
69
+ ssl true
70
+ ]
71
+ d = create_driver(conf)
72
+ expected = {
73
+ write: {
74
+ j: false,
75
+ },
76
+ ssl: true,
77
+ ssl_cert: nil,
78
+ ssl_key: nil,
79
+ ssl_key_pass_phrase: nil,
80
+ ssl_verify: false,
81
+ ssl_ca_cert: nil,
82
+ }
83
+ assert_equal(expected, d.instance.client_options)
84
+ end
85
+
86
+ def test_configure_with_tag_mapped
87
+ conf = default_config + %[
88
+ tag_mapped true
89
+ remove_tag_prefix raw.
90
+ ]
91
+ d = create_driver(conf)
92
+ assert_true(d.instance.tag_mapped)
93
+ assert_equal(/^raw\./, d.instance.remove_tag_prefix)
94
+ end
95
+
96
+ def test_configure_with_write_concern
97
+ d = create_driver(default_config + %[
98
+ write_concern 2
99
+ ])
100
+
101
+ expected = {
102
+ ssl: false,
103
+ write: {
104
+ j: false,
105
+ w: 2,
106
+ },
107
+ }
108
+ assert_equal(expected, d.instance.client_options)
109
+ end
110
+
111
+ def test_configure_with_journaled
112
+ d = create_driver(default_config + %[
113
+ journaled true
114
+ ])
115
+
116
+ expected = {
117
+ ssl: false,
118
+ write: {
119
+ j: true,
120
+ },
121
+ }
122
+ assert_equal(expected, d.instance.client_options)
123
+ end
124
+
125
+ def test_configure_with_logger_conf
126
+ d = create_driver(default_config + %[
127
+ mongo_log_level fatal
128
+ ])
129
+
130
+ expected = "fatal"
131
+ assert_equal(expected, d.instance.mongo_log_level)
132
+ end
133
+
134
+ def get_documents
135
+ @client[collection_name].find.to_a.map {|e| e.delete('_id'); e}
136
+ end
137
+
138
+ def emit_documents(d)
139
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
140
+ d.emit({'a' => 1}, time)
141
+ d.emit({'a' => 2}, time)
142
+ time
143
+ end
144
+
145
+ def test_format
146
+ d = create_driver
147
+
148
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
149
+ d.emit({'a' => 1}, time)
150
+ d.emit({'a' => 2}, time)
151
+ d.expect_format([time, {'a' => 1, d.instance.time_key => time}].to_msgpack)
152
+ d.expect_format([time, {'a' => 2, d.instance.time_key => time}].to_msgpack)
153
+ d.run
154
+
155
+ documents = get_documents
156
+ assert_equal(2, documents.size)
157
+ end
158
+
159
+ def test_write
160
+ d = create_driver
161
+ t = emit_documents(d)
162
+
163
+ d.run
164
+ actual_documents = get_documents
165
+ time = Time.parse("2011-01-02 13:14:15 UTC")
166
+ expected = [{'a' => 1, d.instance.time_key => time},
167
+ {'a' => 2, d.instance.time_key => time}]
168
+ assert_equal(expected, actual_documents)
169
+ end
170
+
171
+ def test_write_at_enable_tag
172
+ d = create_driver(default_config + %[
173
+ include_tag_key true
174
+ include_time_key false
175
+ ])
176
+ t = emit_documents(d)
177
+
178
+ d.run
179
+ actual_documents = get_documents
180
+ expected = [{'a' => 1, d.instance.tag_key => 'test'},
181
+ {'a' => 2, d.instance.tag_key => 'test'}]
182
+ assert_equal(expected, actual_documents)
183
+ end
184
+
185
+ def emit_invalid_documents(d)
186
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
187
+ d.emit({'a' => 3, 'b' => "c", '$last' => '石動'}, time)
188
+ d.emit({'a' => 4, 'b' => "d", 'first' => '菖蒲'.encode('EUC-JP').force_encoding('UTF-8')}, time)
189
+ time
190
+ end
191
+
192
+ def test_write_with_invalid_recoreds_with_keys_containing_dot_and_dollar
193
+ d = create_driver(default_config + %[
194
+ replace_dot_in_key_with _dot_
195
+ replace_dollar_in_key_with _dollar_
196
+ ])
197
+
198
+ original_time = "2016-02-01 13:14:15 UTC"
199
+ time = Time.parse(original_time).to_i
200
+ d.emit({
201
+ "foo.bar1" => {
202
+ "$foo$bar" => "baz"
203
+ },
204
+ "foo.bar2" => [
205
+ {
206
+ "$foo$bar" => "baz"
207
+ }
208
+ ],
209
+ }, time)
210
+ d.run
211
+
212
+ documents = get_documents
213
+ expected = {"foo_dot_bar1" => {
214
+ "_dollar_foo$bar"=>"baz"
215
+ },
216
+ "foo_dot_bar2" => [
217
+ {
218
+ "_dollar_foo$bar"=>"baz"
219
+ },
220
+ ], "time" => Time.parse(original_time)
221
+ }
222
+ assert_equal(1, documents.size)
223
+ assert_equal(expected, documents[0])
224
+ end
225
+
226
+ class WithAuthenticateTest < self
227
+ def setup_mongod
228
+ options = {}
229
+ options[:database] = database_name
230
+ @client = ::Mongo::Client.new(["localhost:#{port}"], options)
231
+ @client.database.users.create('fluent', password: 'password',
232
+ roles: [Mongo::Auth::Roles::READ_WRITE])
233
+ end
234
+
235
+ def teardown_mongod
236
+ @client[collection_name].drop
237
+ @client.database.users.remove('fluent')
238
+ end
239
+
240
+ def test_write_with_authenticate
241
+ d = create_driver(default_config + %[
242
+ user fluent
243
+ password password
244
+ ])
245
+ t = emit_documents(d)
246
+
247
+ d.run
248
+ actual_documents = get_documents
249
+ time = Time.parse("2011-01-02 13:14:15 UTC")
250
+ expected = [{'a' => 1, d.instance.time_key => time},
251
+ {'a' => 2, d.instance.time_key => time}]
252
+ assert_equal(expected, actual_documents)
253
+ end
254
+ end
255
+
256
+ class MongoAuthenticateTest < self
257
+ require 'fluent/plugin/mongo_auth'
258
+ include ::Fluent::MongoAuth
259
+
260
+ def setup_mongod
261
+ options = {}
262
+ options[:database] = database_name
263
+ @client = ::Mongo::Client.new(["localhost:#{port}"], options)
264
+ @client.database.users.create('fluent', password: 'password',
265
+ roles: [Mongo::Auth::Roles::READ_WRITE])
266
+ end
267
+
268
+ def teardown_mongod
269
+ @client[collection_name].drop
270
+ @client.database.users.remove('fluent')
271
+ end
272
+
273
+ def test_authenticate
274
+ d = create_driver(default_config + %[
275
+ user fluent
276
+ password password
277
+ ])
278
+
279
+ assert authenticate(@client)
280
+ end
281
+ end
282
+ end
@@ -0,0 +1,125 @@
1
+ require "helper"
2
+
3
+ class MongoReplsetOutputTest < ::Test::Unit::TestCase
4
+ def setup
5
+ Fluent::Test.setup
6
+ end
7
+
8
+ def teardown
9
+ end
10
+
11
+ def collection_name
12
+ 'test'
13
+ end
14
+
15
+ def database_name
16
+ 'fluent_test'
17
+ end
18
+
19
+ def port
20
+ 27018
21
+ end
22
+
23
+ def default_config
24
+ %[
25
+ type mongo
26
+ port 27018
27
+ database #{database_name}
28
+ collection #{collection_name}
29
+ include_time_key true
30
+ replica_set rs0
31
+ ]
32
+ end
33
+
34
+ def create_driver(conf=default_config, tag='test')
35
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::MongoOutputReplset, tag).configure(conf)
36
+ end
37
+
38
+ def test_configure
39
+ d = create_driver(%[
40
+ type mongo
41
+ port 27018
42
+ database fluent_test
43
+ collection test_collection
44
+
45
+ replica_set rs0
46
+ ])
47
+
48
+ assert_equal('fluent_test', d.instance.database)
49
+ assert_equal('test_collection', d.instance.collection)
50
+ assert_equal('localhost', d.instance.host)
51
+ assert_equal(port, d.instance.port)
52
+ assert_equal({replica_set: 'rs0', :ssl=>false, :write=>{:j=>false}},
53
+ d.instance.client_options)
54
+ end
55
+
56
+ def test_configure_with_logger_conf
57
+ d = create_driver(default_config + %[
58
+ mongo_log_level fatal
59
+ ])
60
+
61
+ expected = "fatal"
62
+ assert_equal(expected, d.instance.mongo_log_level)
63
+ end
64
+
65
+ class ReplisetWriteTest < self
66
+ def setup
67
+ omit("Replica set setup is too hard in CI.") if ENV['CI']
68
+
69
+ setup_mongod
70
+ end
71
+
72
+ def teardown
73
+ omit("Replica set setup is too hard in CI.") if ENV['CI']
74
+
75
+ teardown_mongod
76
+ end
77
+
78
+ def setup_mongod
79
+ options = {}
80
+ options[:database] = database_name
81
+ @client = ::Mongo::Client.new(["localhost:#{port}"], options)
82
+ end
83
+
84
+ def teardown_mongod
85
+ @client[collection_name].drop
86
+ end
87
+
88
+ def get_documents
89
+ @client[collection_name].find.to_a.map {|e| e.delete('_id'); e}
90
+ end
91
+
92
+ def emit_documents(d)
93
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
94
+ d.emit({'a' => 1}, time)
95
+ d.emit({'a' => 2}, time)
96
+ time
97
+ end
98
+
99
+ def test_format
100
+ d = create_driver
101
+
102
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
103
+ d.emit({'a' => 1}, time)
104
+ d.emit({'a' => 2}, time)
105
+ d.expect_format([time, {'a' => 1, d.instance.time_key => time}].to_msgpack)
106
+ d.expect_format([time, {'a' => 2, d.instance.time_key => time}].to_msgpack)
107
+ d.run
108
+
109
+ documents = get_documents
110
+ assert_equal(2, documents.size)
111
+ end
112
+
113
+ def test_write
114
+ d = create_driver
115
+ t = emit_documents(d)
116
+
117
+ d.run
118
+ actual_documents = get_documents
119
+ time = Time.parse("2011-01-02 13:14:15 UTC")
120
+ expected = [{'a' => 1, d.instance.time_key => time},
121
+ {'a' => 2, d.instance.time_key => time}]
122
+ assert_equal(expected, actual_documents)
123
+ end
124
+ end
125
+ end