fluent-plugin-mongo 1.3.0 → 1.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: fb5c56a40c5a058bd8b710af7fe2b065b96b2e0d
4
- data.tar.gz: 761f54a14e8dde03a9c669ff89a0dda9d07b84ce
2
+ SHA256:
3
+ metadata.gz: 709f3068ce7f97bcad4737b3c7a9db60bdea998b184949afa7f3fa1794831474
4
+ data.tar.gz: 53d8b989b2d30d8b0df0e37701af671713460dd46fe920435016ececcaef6c36
5
5
  SHA512:
6
- metadata.gz: b9b5368add232fe67d644a95d28e5adbe8f4584f63e67c347effb36b50188b1321f1628d16c4494c1aac404db120edc91ff09558f85ad3a7439310909fc4e4d9
7
- data.tar.gz: cd1d6a83f39438d464829baab15d6e9e490fa1b693947111b9ffc7176ff2c8c73673f5c109a562a9d17be447b99cbf59fbb9ebf98ebc04f6ba39cdaafcdb4684
6
+ metadata.gz: 6ce121ba68754c2d6687a7d1ed8cedf8c9fe94652f85476b03aacddf92bede132aa7f45444c8cb4153ff5509143bf00688cffc5c872c15e76c754e0cd6f5ff8f
7
+ data.tar.gz: 4391ce8e22ca8173f38630a6f22024acdccae01d48a3bf5c86f255bbdbe057cdbbb9039b9f55b209bfaf377188124a5b1dd83fcaf8b58db80785e16348e8caa0
@@ -1,9 +1,8 @@
1
1
  rvm:
2
- - 2.1
3
- - 2.2.4
4
- - 2.3.1
5
- - 2.4.3
6
- - 2.5.0
2
+ - 2.4.10
3
+ - 2.5
4
+ - 2.6
5
+ - 2.7
7
6
  - ruby-head
8
7
 
9
8
  gemfile:
@@ -12,12 +11,6 @@ gemfile:
12
11
  services:
13
12
  - mongodb
14
13
 
15
- before_install:
16
- - gem update --system=2.7.8
17
-
18
- before_script:
19
- - git submodule update -i
20
-
21
14
  script: bundle exec rake test
22
15
 
23
16
  matrix:
data/ChangeLog CHANGED
@@ -1,3 +1,7 @@
1
+ Release 1.4.0 - 2020/04/28
2
+
3
+ * out_mongo: Add date_keys parameter to support MongoDB Date object in record fields.
4
+
1
5
  Release 1.3.0 - 2019/04/24
2
6
 
3
7
  * out_mongo: Support auth_mech parameter to allow other authentication
@@ -53,6 +53,12 @@ Use _mongo_ type in match.
53
53
  capped
54
54
  capped_size 100m
55
55
 
56
+ # Specify date fields in record to use MongoDB's Date object (Optional) default: nil
57
+ # Supported data types are String/Integer/Float/Fuentd EventTime.
58
+ # For Integer type, milliseconds epoch and seconds epoch are supported.
59
+ # eg: updated_at: "2020-02-01T08:22:23.780Z" or updated_at: 1580546457010
60
+ date_keys updated_at
61
+
56
62
  # Other buffer configurations here
57
63
  </match>
58
64
 
@@ -223,7 +229,7 @@ BSON records which include '.' or start with '$' are invalid and they will be st
223
229
  ...
224
230
  # replace '.' in keys with '__dot__'
225
231
  replace_dot_in_key_with __dot__
226
-
232
+
227
233
  # replace '$' in keys with '__dollar__'
228
234
  # Note: This replaces '$' only on first character
229
235
  replace_dollar_in_key_with __dollar__
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.3.0
1
+ 1.4.0
@@ -10,7 +10,6 @@ Gem::Specification.new do |gem|
10
10
  gem.version = File.read("VERSION").strip
11
11
  gem.authors = ["Masahiro Nakagawa"]
12
12
  gem.email = "repeatedly@gmail.com"
13
- gem.has_rdoc = false
14
13
  #gem.platform = Gem::Platform::RUBY
15
14
  gem.files = `git ls-files`.split("\n")
16
15
  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
@@ -38,6 +38,10 @@ module Fluent::Plugin
38
38
  desc "Replace dollar with specified string"
39
39
  config_param :replace_dollar_in_key_with, :string, default: nil
40
40
 
41
+ # Additional date field to be used to Date object
42
+ desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
43
+ config_param :date_keys, :array, default: nil
44
+
41
45
  # tag mapping mode
42
46
  desc "Use tag_mapped mode"
43
47
  config_param :tag_mapped, :bool, default: false,
@@ -54,6 +58,7 @@ module Fluent::Plugin
54
58
  config_param :ssl_verify, :bool, default: false
55
59
  config_param :ssl_ca_cert, :string, default: nil
56
60
 
61
+
57
62
  config_section :buffer do
58
63
  config_set_default :@type, DEFAULT_BUFFER_TYPE
59
64
  config_set_default :chunk_keys, ['tag']
@@ -198,11 +203,40 @@ module Fluent::Plugin
198
203
  def collect_records(chunk)
199
204
  records = []
200
205
  time_key = @inject_config.time_key if @inject_config
206
+ date_keys = @date_keys
207
+
201
208
  tag = chunk.metadata.tag
202
209
  chunk.msgpack_each {|time, record|
203
210
  record = inject_values_to_record(tag, time, record)
204
211
  # MongoDB uses BSON's Date for time.
205
212
  record[time_key] = Time.at(time || record[time_key]) if time_key
213
+
214
+ if date_keys
215
+ date_keys.each { |date_key|
216
+ begin
217
+ date_value = record[date_key]
218
+ case date_value
219
+ when Fluent::EventTime
220
+ record[date_key] = date_value.to_time
221
+ when Integer
222
+ record[date_key] = if date_value > 9999999999
223
+ # epoch with milliseconds: e.g. javascript
224
+ Time.at(date_value / 1000.0)
225
+ else
226
+ # epoch with seconds: e.g. ruby
227
+ Time.at(date_value)
228
+ end
229
+ when Float
230
+ record[date_key] = Time.at(date_value)
231
+ else
232
+ record[date_key] = Time.parse(date_value)
233
+ end
234
+ rescue ArgumentError
235
+ log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{record[date_key]}"
236
+ record[date_key] = nil
237
+ end
238
+ }
239
+ end
206
240
  records << record
207
241
  }
208
242
  records
@@ -100,7 +100,7 @@ class MongoTailInputTest < Test::Unit::TestCase
100
100
  end
101
101
  events = d.events
102
102
  assert_equal "input.mongo", events[0][0]
103
- assert_equal event_time(@time.to_s), events[0][1]
103
+ assert events[0][1].is_a?(Fluent::EventTime)
104
104
  assert_equal "test", events[0][2]["message"]
105
105
  end
106
106
 
@@ -138,7 +138,7 @@ class MongoTailInputTest < Test::Unit::TestCase
138
138
  events = d.events
139
139
  assert_equal 1, events.size
140
140
  assert_equal "input.mongo.last_id", events[0][0]
141
- assert_equal event_time(@time.to_s), events[0][1]
141
+ assert events[0][1].is_a?(Fluent::EventTime)
142
142
  assert_equal "can obtain", events[0][2]["message"]
143
143
  end
144
144
  end
@@ -315,7 +315,7 @@ class MongoOutputTest < ::Test::Unit::TestCase
315
315
  "$foo$bar" => "baz"
316
316
  }
317
317
  ],
318
- })
318
+ })
319
319
  end
320
320
 
321
321
  documents = get_documents
@@ -388,4 +388,56 @@ class MongoOutputTest < ::Test::Unit::TestCase
388
388
  assert authenticate(@client)
389
389
  end
390
390
  end
391
+
392
+ sub_test_case 'date_keys' do
393
+ setup do
394
+ @updated_at_str = "2020-02-01T08:22:23.780Z"
395
+ @updated_at_t = Time.parse(@updated_at_str)
396
+ end
397
+
398
+ def emit_date_documents(d)
399
+ time = event_time("2011-01-02 13:14:15 UTC")
400
+ d.feed(time, {'a' => 1, updated_at: @updated_at_str})
401
+ d.feed(time, {'a' => 2, updated_at: @updated_at_t.to_f})
402
+ d.feed(time, {'a' => 3, updated_at: @updated_at_t.to_i})
403
+ time
404
+ end
405
+
406
+ def emit_invalid_date_documents(d)
407
+ time = event_time("2011-01-02 13:14:15 UTC")
408
+ d.feed(time, {'a' => 1, updated_at: "Invalid Date String"})
409
+ time
410
+ end
411
+
412
+ def test_write_with_date_keys
413
+ d = create_driver(default_config + %[
414
+ date_keys updated_at
415
+ time_key created_at
416
+ ])
417
+
418
+ d.run(default_tag: 'test') do
419
+ emit_date_documents(d)
420
+ end
421
+
422
+ actual_documents = get_documents
423
+ date_key = d.instance.date_keys.first
424
+ actual_documents.each_with_index { |doc, i|
425
+ assert_equal(i + 1, doc['a'])
426
+ assert doc[date_key].is_a?(Time)
427
+ }
428
+ end
429
+
430
+ def test_write_with_parsed_date_key_invalid_string
431
+ d = create_driver(default_config + %[
432
+ date_keys updated_at
433
+ time_key created_at
434
+ ])
435
+
436
+ d.run(default_tag: 'test') do
437
+ emit_invalid_date_documents(d)
438
+ end
439
+ actual_documents = get_documents
440
+ assert_nil actual_documents.first['updated_at']
441
+ end
442
+ end
391
443
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-mongo
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro Nakagawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-04-24 00:00:00.000000000 Z
11
+ date: 2020-04-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -159,8 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
159
159
  - !ruby/object:Gem::Version
160
160
  version: '0'
161
161
  requirements: []
162
- rubyforge_project:
163
- rubygems_version: 2.6.14.1
162
+ rubygems_version: 3.0.3
164
163
  signing_key:
165
164
  specification_version: 4
166
165
  summary: MongoDB plugin for Fluentd