fluent-plugin-mongo 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +4 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/out_mongo.rb +42 -13
- data/test/plugin/test_out_mongo.rb +48 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6208365e5862e84db04e62061f822324bc6ce1f82ebb8b9b0c3cb6ed230d983e
|
4
|
+
data.tar.gz: 2c45c015dd0fc2cf98e3b309ec4d96865762ed6bf6e534d316f5c6b214c55fcb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 046b7463bad7a1be9aae5f2a20991656392e32a4693de753aa0c5010f269d8708e0a2c085b5809408b17c5114aeff534e5cafe5b2527453a07f6707227bcbd15
|
7
|
+
data.tar.gz: 42d7e9a95afeb58709422a13e0e90a911a6e32b8d972e26982496b5e90c106db83eb7d5b06d6c775a4f16a92569f10f2e82764a26d20c1c061408ad2bee89cb9
|
data/ChangeLog
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.5.0
|
@@ -8,7 +8,7 @@ module Fluent::Plugin
|
|
8
8
|
class MongoOutput < Output
|
9
9
|
Fluent::Plugin.register_output('mongo', self)
|
10
10
|
|
11
|
-
helpers :event_emitter, :inject, :compat_parameters
|
11
|
+
helpers :event_emitter, :inject, :compat_parameters, :record_accessor
|
12
12
|
|
13
13
|
include Fluent::MongoAuthParams
|
14
14
|
include Fluent::MongoAuth
|
@@ -41,6 +41,9 @@ module Fluent::Plugin
|
|
41
41
|
# Additional date field to be used to Date object
|
42
42
|
desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
|
43
43
|
config_param :date_keys, :array, default: nil
|
44
|
+
desc "Specify if the fields in date_keys are of type Integer or Float"
|
45
|
+
config_param :parse_string_number_date, :bool, default: false
|
46
|
+
|
44
47
|
|
45
48
|
# tag mapping mode
|
46
49
|
desc "Use tag_mapped mode"
|
@@ -75,6 +78,7 @@ module Fluent::Plugin
|
|
75
78
|
@nodes = nil
|
76
79
|
@client_options = {}
|
77
80
|
@collection_options = {capped: false}
|
81
|
+
@accessors = {}
|
78
82
|
end
|
79
83
|
|
80
84
|
# Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
|
@@ -157,6 +161,13 @@ module Fluent::Plugin
|
|
157
161
|
configure_logger(@mongo_log_level)
|
158
162
|
|
159
163
|
log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
|
164
|
+
|
165
|
+
if @date_keys
|
166
|
+
@date_keys.each { |field_name|
|
167
|
+
@accessors[field_name.to_s] = record_accessor_create(field_name)
|
168
|
+
}
|
169
|
+
log.debug "Setup record accessor for every date key"
|
170
|
+
end
|
160
171
|
end
|
161
172
|
|
162
173
|
def start
|
@@ -211,28 +222,46 @@ module Fluent::Plugin
|
|
211
222
|
record[time_key] = Time.at(time || record[time_key]) if time_key
|
212
223
|
|
213
224
|
if date_keys
|
214
|
-
|
225
|
+
@accessors.each_pair { |date_key, date_key_accessor|
|
215
226
|
begin
|
216
|
-
date_value = record
|
227
|
+
date_value = date_key_accessor.call(record)
|
217
228
|
case date_value
|
218
229
|
when Fluent::EventTime
|
219
|
-
|
230
|
+
value_to_set = date_value.to_time
|
220
231
|
when Integer
|
221
|
-
|
232
|
+
value_to_set = if date_value > 9999999999
|
233
|
+
# epoch with milliseconds: e.g. javascript
|
234
|
+
Time.at(date_value / 1000.0)
|
235
|
+
else
|
236
|
+
# epoch with seconds: e.g. ruby
|
237
|
+
Time.at(date_value)
|
238
|
+
end
|
239
|
+
when Float
|
240
|
+
value_to_set = Time.at(date_value)
|
241
|
+
else
|
242
|
+
if @parse_string_number_date
|
243
|
+
if date_value.to_i.to_s == date_value
|
244
|
+
date_value = date_value.to_i
|
245
|
+
value_to_set = if date_value > 9999999999
|
222
246
|
# epoch with milliseconds: e.g. javascript
|
223
|
-
|
247
|
+
date_value / 1000.0
|
224
248
|
else
|
225
249
|
# epoch with seconds: e.g. ruby
|
226
|
-
|
250
|
+
date_value
|
227
251
|
end
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
252
|
+
elsif date_value.to_f.to_s == date_value
|
253
|
+
date_value = date_value.to_f
|
254
|
+
end
|
255
|
+
value_to_set = date_value.is_a?(String) ? Time.parse(date_value) : Time.at(date_value)
|
256
|
+
else
|
257
|
+
value_to_set = Time.parse(date_value)
|
258
|
+
end
|
232
259
|
end
|
260
|
+
|
261
|
+
date_key_accessor.set(record, value_to_set)
|
233
262
|
rescue ArgumentError
|
234
|
-
log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{
|
235
|
-
record
|
263
|
+
log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{date_value}"
|
264
|
+
date_key_accessor.set(record, nil)
|
236
265
|
end
|
237
266
|
}
|
238
267
|
end
|
@@ -438,6 +438,20 @@ class MongoOutputTest < ::Test::Unit::TestCase
|
|
438
438
|
time
|
439
439
|
end
|
440
440
|
|
441
|
+
def emit_nested_date_documents(d)
|
442
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
443
|
+
d.feed(time, {'a' => 1, updated_at: { 'time': @updated_at_str}})
|
444
|
+
d.feed(time, {'a' => 2, updated_at: { 'time': @updated_at_t.to_f}})
|
445
|
+
d.feed(time, {'a' => 3, updated_at: { 'time': @updated_at_t.to_i}})
|
446
|
+
time
|
447
|
+
end
|
448
|
+
|
449
|
+
def emit_nested_invalid_date_documents(d)
|
450
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
451
|
+
d.feed(time, {'a' => 1, 'updated_at': { 'time': "Invalid Date String"}})
|
452
|
+
time
|
453
|
+
end
|
454
|
+
|
441
455
|
def test_write_with_date_keys
|
442
456
|
d = create_driver(default_config + %[
|
443
457
|
date_keys updated_at
|
@@ -468,5 +482,39 @@ class MongoOutputTest < ::Test::Unit::TestCase
|
|
468
482
|
actual_documents = get_documents
|
469
483
|
assert_nil actual_documents.first['updated_at']
|
470
484
|
end
|
485
|
+
|
486
|
+
def test_write_with_date_nested_keys
|
487
|
+
d = create_driver(default_config + %[
|
488
|
+
replace_dot_in_key_with _
|
489
|
+
replace_dollar_in_key_with _
|
490
|
+
date_keys $.updated_at.time
|
491
|
+
time_key created_at
|
492
|
+
])
|
493
|
+
|
494
|
+
d.run(default_tag: 'test') do
|
495
|
+
emit_nested_date_documents(d)
|
496
|
+
end
|
497
|
+
|
498
|
+
actual_documents = get_documents
|
499
|
+
actual_documents.each_with_index { |doc, i|
|
500
|
+
assert_equal(i + 1, doc['a'])
|
501
|
+
assert doc['updated_at']['time'].is_a?(Time)
|
502
|
+
}
|
503
|
+
end
|
504
|
+
|
505
|
+
def test_write_with_parsed_date_nested_key_invalid_string
|
506
|
+
d = create_driver(default_config + %[
|
507
|
+
replace_dot_in_key_with _
|
508
|
+
replace_dollar_in_key_with _
|
509
|
+
date_keys $.updated_at.time
|
510
|
+
time_key created_at
|
511
|
+
])
|
512
|
+
|
513
|
+
d.run(default_tag: 'test') do
|
514
|
+
emit_nested_invalid_date_documents(d)
|
515
|
+
end
|
516
|
+
actual_documents = get_documents
|
517
|
+
assert_nil actual_documents.first['updated_at']['time']
|
518
|
+
end
|
471
519
|
end
|
472
520
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mongo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|