fluent-plugin-mongo 1.4.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: baad354dcd89624e9e6ed224718d569de6a37329e7a0339d2b8f41011ba0b970
4
- data.tar.gz: 15c2c50509c70a325fe6c053a09062e5701e61003e550d316ffa491e9cc99b75
3
+ metadata.gz: 6208365e5862e84db04e62061f822324bc6ce1f82ebb8b9b0c3cb6ed230d983e
4
+ data.tar.gz: 2c45c015dd0fc2cf98e3b309ec4d96865762ed6bf6e534d316f5c6b214c55fcb
5
5
  SHA512:
6
- metadata.gz: 59aeae091430fae5bb089245f24d85e8e204a11f7d886f0b53b7cde6af054c0e5b9c2fc94f17f2b0dfdc7fb79c562be7112adfafae71583e8fcf8b18aa8766ab
7
- data.tar.gz: 242c5d3d7a58f0be87514909749b066942118d90c24e9990f719eff5956919556a86ae24c427c8d21e55b70b39d03b8796d10c4493d00d7c41a7be635009f738
6
+ metadata.gz: 046b7463bad7a1be9aae5f2a20991656392e32a4693de753aa0c5010f269d8708e0a2c085b5809408b17c5114aeff534e5cafe5b2527453a07f6707227bcbd15
7
+ data.tar.gz: 42d7e9a95afeb58709422a13e0e90a911a6e32b8d972e26982496b5e90c106db83eb7d5b06d6c775a4f16a92569f10f2e82764a26d20c1c061408ad2bee89cb9
data/ChangeLog CHANGED
@@ -1,3 +1,7 @@
1
+ Release 1.5.0 - 2020/11/26
2
+
3
+ * out_mongo: Support nested fields in date_keys
4
+
1
5
  Release 1.4.1 - 2020/08/21
2
6
 
3
7
  * out_mongo: Add expire_after parameter
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.4.1
1
+ 1.5.0
@@ -8,7 +8,7 @@ module Fluent::Plugin
8
8
  class MongoOutput < Output
9
9
  Fluent::Plugin.register_output('mongo', self)
10
10
 
11
- helpers :event_emitter, :inject, :compat_parameters
11
+ helpers :event_emitter, :inject, :compat_parameters, :record_accessor
12
12
 
13
13
  include Fluent::MongoAuthParams
14
14
  include Fluent::MongoAuth
@@ -41,6 +41,9 @@ module Fluent::Plugin
41
41
  # Additional date field to be used to Date object
42
42
  desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
43
43
  config_param :date_keys, :array, default: nil
44
+ desc "Specify if the fields in date_keys are of type Integer or Float"
45
+ config_param :parse_string_number_date, :bool, default: false
46
+
44
47
 
45
48
  # tag mapping mode
46
49
  desc "Use tag_mapped mode"
@@ -75,6 +78,7 @@ module Fluent::Plugin
75
78
  @nodes = nil
76
79
  @client_options = {}
77
80
  @collection_options = {capped: false}
81
+ @accessors = {}
78
82
  end
79
83
 
80
84
  # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
@@ -157,6 +161,13 @@ module Fluent::Plugin
157
161
  configure_logger(@mongo_log_level)
158
162
 
159
163
  log.debug "Setup mongo configuration: mode = #{@tag_mapped ? 'tag mapped' : 'normal'}"
164
+
165
+ if @date_keys
166
+ @date_keys.each { |field_name|
167
+ @accessors[field_name.to_s] = record_accessor_create(field_name)
168
+ }
169
+ log.debug "Setup record accessor for every date key"
170
+ end
160
171
  end
161
172
 
162
173
  def start
@@ -211,28 +222,46 @@ module Fluent::Plugin
211
222
  record[time_key] = Time.at(time || record[time_key]) if time_key
212
223
 
213
224
  if date_keys
214
- date_keys.each { |date_key|
225
+ @accessors.each_pair { |date_key, date_key_accessor|
215
226
  begin
216
- date_value = record[date_key]
227
+ date_value = date_key_accessor.call(record)
217
228
  case date_value
218
229
  when Fluent::EventTime
219
- record[date_key] = date_value.to_time
230
+ value_to_set = date_value.to_time
220
231
  when Integer
221
- record[date_key] = if date_value > 9999999999
232
+ value_to_set = if date_value > 9999999999
233
+ # epoch with milliseconds: e.g. javascript
234
+ Time.at(date_value / 1000.0)
235
+ else
236
+ # epoch with seconds: e.g. ruby
237
+ Time.at(date_value)
238
+ end
239
+ when Float
240
+ value_to_set = Time.at(date_value)
241
+ else
242
+ if @parse_string_number_date
243
+ if date_value.to_i.to_s == date_value
244
+ date_value = date_value.to_i
245
+ value_to_set = if date_value > 9999999999
222
246
  # epoch with milliseconds: e.g. javascript
223
- Time.at(date_value / 1000.0)
247
+ date_value / 1000.0
224
248
  else
225
249
  # epoch with seconds: e.g. ruby
226
- Time.at(date_value)
250
+ date_value
227
251
  end
228
- when Float
229
- record[date_key] = Time.at(date_value)
230
- else
231
- record[date_key] = Time.parse(date_value)
252
+ elsif date_value.to_f.to_s == date_value
253
+ date_value = date_value.to_f
254
+ end
255
+ value_to_set = date_value.is_a?(String) ? Time.parse(date_value) : Time.at(date_value)
256
+ else
257
+ value_to_set = Time.parse(date_value)
258
+ end
232
259
  end
260
+
261
+ date_key_accessor.set(record, value_to_set)
233
262
  rescue ArgumentError
234
- log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{record[date_key]}"
235
- record[date_key] = nil
263
+ log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{date_value}"
264
+ date_key_accessor.set(record, nil)
236
265
  end
237
266
  }
238
267
  end
@@ -438,6 +438,20 @@ class MongoOutputTest < ::Test::Unit::TestCase
438
438
  time
439
439
  end
440
440
 
441
+ def emit_nested_date_documents(d)
442
+ time = event_time("2011-01-02 13:14:15 UTC")
443
+ d.feed(time, {'a' => 1, updated_at: { 'time': @updated_at_str}})
444
+ d.feed(time, {'a' => 2, updated_at: { 'time': @updated_at_t.to_f}})
445
+ d.feed(time, {'a' => 3, updated_at: { 'time': @updated_at_t.to_i}})
446
+ time
447
+ end
448
+
449
+ def emit_nested_invalid_date_documents(d)
450
+ time = event_time("2011-01-02 13:14:15 UTC")
451
+ d.feed(time, {'a' => 1, 'updated_at': { 'time': "Invalid Date String"}})
452
+ time
453
+ end
454
+
441
455
  def test_write_with_date_keys
442
456
  d = create_driver(default_config + %[
443
457
  date_keys updated_at
@@ -468,5 +482,39 @@ class MongoOutputTest < ::Test::Unit::TestCase
468
482
  actual_documents = get_documents
469
483
  assert_nil actual_documents.first['updated_at']
470
484
  end
485
+
486
+ def test_write_with_date_nested_keys
487
+ d = create_driver(default_config + %[
488
+ replace_dot_in_key_with _
489
+ replace_dollar_in_key_with _
490
+ date_keys $.updated_at.time
491
+ time_key created_at
492
+ ])
493
+
494
+ d.run(default_tag: 'test') do
495
+ emit_nested_date_documents(d)
496
+ end
497
+
498
+ actual_documents = get_documents
499
+ actual_documents.each_with_index { |doc, i|
500
+ assert_equal(i + 1, doc['a'])
501
+ assert doc['updated_at']['time'].is_a?(Time)
502
+ }
503
+ end
504
+
505
+ def test_write_with_parsed_date_nested_key_invalid_string
506
+ d = create_driver(default_config + %[
507
+ replace_dot_in_key_with _
508
+ replace_dollar_in_key_with _
509
+ date_keys $.updated_at.time
510
+ time_key created_at
511
+ ])
512
+
513
+ d.run(default_tag: 'test') do
514
+ emit_nested_invalid_date_documents(d)
515
+ end
516
+ actual_documents = get_documents
517
+ assert_nil actual_documents.first['updated_at']['time']
518
+ end
471
519
  end
472
520
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mongo
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.1
4
+ version: 1.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro Nakagawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-08-21 00:00:00.000000000 Z
11
+ date: 2020-11-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd