fluent-plugin-mongo 1.1.2 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/ChangeLog +4 -0
- data/README.rdoc +24 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_mongo.rb +8 -6
- data/test/plugin/test_out_mongo.rb +31 -2
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ae5027563d8db12e0cc4036dec85ca15dd6d5d19
|
4
|
+
data.tar.gz: 93fde5d241a4b22bc89a3a13bee779e2ed06faea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fa962dc7b06fa64e3ca1ee33c20294327f09f21adfc03e89d43231adfe167608711dbe617becedc1e2ea3e353337126b4cbae8663ca560372a82d3b0a3af3d15
|
7
|
+
data.tar.gz: ba9202858bbfa8259fed50d3d2ecc879d612f18973fd1b5cedc66193e8747c75218a9cfceb225f0164e121f552e9c88727ee7df1c996f16d1dae282d33ff9517
|
data/ChangeLog
CHANGED
data/README.rdoc
CHANGED
@@ -58,6 +58,30 @@ Use _mongo_ type in match.
|
|
58
58
|
|
59
59
|
For _connection_string_ parameter, see https://docs.mongodb.com/manual/reference/connection-string/ article for more detail.
|
60
60
|
|
61
|
+
===== built-in placeholders
|
62
|
+
|
63
|
+
fluent-plugin-mongo support built-in placeholders.
|
64
|
+
_database_ and _collection_ parameters can handle them.
|
65
|
+
|
66
|
+
Here is an example to use built-in placeholders:
|
67
|
+
|
68
|
+
<match mongo.**>
|
69
|
+
@type mongo
|
70
|
+
|
71
|
+
database ${tag[0]}
|
72
|
+
|
73
|
+
# collection name to insert
|
74
|
+
collection ${tag[1]}-%Y%m%d
|
75
|
+
|
76
|
+
# Other buffer configurations here
|
77
|
+
<buffer tag, time>
|
78
|
+
@type memory
|
79
|
+
timekey 3600
|
80
|
+
</buffer>
|
81
|
+
</match>
|
82
|
+
|
83
|
+
In more detail, please refer to the officilal document for built-in placeholders: https://docs.fluentd.org/v1.0/articles/buffer-section#placeholders
|
84
|
+
|
61
85
|
=== mongo(tag mapped mode)
|
62
86
|
|
63
87
|
Tag mapped to MongoDB collection automatically.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.2.0
|
@@ -173,16 +173,17 @@ module Fluent::Plugin
|
|
173
173
|
|
174
174
|
def write(chunk)
|
175
175
|
collection_name = extract_placeholders(@collection, chunk.metadata)
|
176
|
-
|
176
|
+
database_name = extract_placeholders(@database, chunk.metadata)
|
177
|
+
operate(database_name, format_collection_name(collection_name), collect_records(chunk))
|
177
178
|
end
|
178
179
|
|
179
180
|
private
|
180
181
|
|
181
|
-
def client
|
182
|
+
def client(database = @database)
|
182
183
|
if @connection_string
|
183
184
|
Mongo::Client.new(@connection_string)
|
184
185
|
else
|
185
|
-
@client_options[:database] =
|
186
|
+
@client_options[:database] = database
|
186
187
|
@client_options[:user] = @user if @user
|
187
188
|
@client_options[:password] = @password if @password
|
188
189
|
Mongo::Client.new(@nodes, @client_options)
|
@@ -227,7 +228,8 @@ module Fluent::Plugin
|
|
227
228
|
end
|
228
229
|
end
|
229
230
|
|
230
|
-
def get_collection(name,
|
231
|
+
def get_collection(database, name, options)
|
232
|
+
@client = client(database) if database && @database != database
|
231
233
|
return @client[name] if @collections[name]
|
232
234
|
|
233
235
|
unless collection_exists?(name)
|
@@ -242,7 +244,7 @@ module Fluent::Plugin
|
|
242
244
|
@collections.delete(name)
|
243
245
|
end
|
244
246
|
|
245
|
-
def operate(collection, records)
|
247
|
+
def operate(database, collection, records)
|
246
248
|
begin
|
247
249
|
if @replace_dot_in_key_with
|
248
250
|
records.map! do |r|
|
@@ -255,7 +257,7 @@ module Fluent::Plugin
|
|
255
257
|
end
|
256
258
|
end
|
257
259
|
|
258
|
-
get_collection(collection, @collection_options).insert_many(records)
|
260
|
+
get_collection(database, collection, @collection_options).insert_many(records)
|
259
261
|
rescue Mongo::Error::BulkWriteError => e
|
260
262
|
log.warn "#{records.size - e.result["n_inserted"]} documents are not inserted. Maybe these documents are invalid as a BSON."
|
261
263
|
forget_collection(collection)
|
@@ -36,9 +36,9 @@ class MongoOutputTest < ::Test::Unit::TestCase
|
|
36
36
|
]
|
37
37
|
end
|
38
38
|
|
39
|
-
def setup_mongod
|
39
|
+
def setup_mongod(database = database_name)
|
40
40
|
options = {}
|
41
|
-
options[:database] =
|
41
|
+
options[:database] = database
|
42
42
|
@client = ::Mongo::Client.new(["localhost:#{port}"], options)
|
43
43
|
end
|
44
44
|
|
@@ -211,6 +211,35 @@ class MongoOutputTest < ::Test::Unit::TestCase
|
|
211
211
|
end
|
212
212
|
end
|
213
213
|
|
214
|
+
class WriteWithDatabasePlaceholder < self
|
215
|
+
def setup
|
216
|
+
@tag = 'custom'
|
217
|
+
setup_mongod(@tag)
|
218
|
+
end
|
219
|
+
|
220
|
+
def teardown
|
221
|
+
teardown_mongod
|
222
|
+
end
|
223
|
+
|
224
|
+
def test_write_with_database_placeholder
|
225
|
+
d = create_driver(%[
|
226
|
+
@type mongo
|
227
|
+
database ${tag}
|
228
|
+
collection #{collection_name}
|
229
|
+
include_time_key true
|
230
|
+
])
|
231
|
+
d.run(default_tag: @tag) do
|
232
|
+
emit_documents(d)
|
233
|
+
end
|
234
|
+
|
235
|
+
actual_documents = get_documents
|
236
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
237
|
+
expected = [{'a' => 1, d.instance.inject_config.time_key => Time.at(time).localtime},
|
238
|
+
{'a' => 2, d.instance.inject_config.time_key => Time.at(time).localtime}]
|
239
|
+
assert_equal(expected, actual_documents)
|
240
|
+
end
|
241
|
+
end
|
242
|
+
|
214
243
|
def test_write_at_enable_tag
|
215
244
|
d = create_driver(default_config + %[
|
216
245
|
include_tag_key true
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mongo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-11-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -160,7 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
160
160
|
version: '0'
|
161
161
|
requirements: []
|
162
162
|
rubyforge_project:
|
163
|
-
rubygems_version: 2.
|
163
|
+
rubygems_version: 2.6.14.1
|
164
164
|
signing_key:
|
165
165
|
specification_version: 4
|
166
166
|
summary: MongoDB plugin for Fluentd
|