fluent-plugin-mongo 1.3.0 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +4 -11
- data/ChangeLog +4 -0
- data/README.rdoc +7 -1
- data/VERSION +1 -1
- data/fluent-plugin-mongo.gemspec +0 -1
- data/lib/fluent/plugin/out_mongo.rb +34 -0
- data/test/plugin/test_in_mongo_tail.rb +2 -2
- data/test/plugin/test_out_mongo.rb +53 -1
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 709f3068ce7f97bcad4737b3c7a9db60bdea998b184949afa7f3fa1794831474
|
4
|
+
data.tar.gz: 53d8b989b2d30d8b0df0e37701af671713460dd46fe920435016ececcaef6c36
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6ce121ba68754c2d6687a7d1ed8cedf8c9fe94652f85476b03aacddf92bede132aa7f45444c8cb4153ff5509143bf00688cffc5c872c15e76c754e0cd6f5ff8f
|
7
|
+
data.tar.gz: 4391ce8e22ca8173f38630a6f22024acdccae01d48a3bf5c86f255bbdbe057cdbbb9039b9f55b209bfaf377188124a5b1dd83fcaf8b58db80785e16348e8caa0
|
data/.travis.yml
CHANGED
@@ -1,9 +1,8 @@
|
|
1
1
|
rvm:
|
2
|
-
- 2.
|
3
|
-
- 2.
|
4
|
-
- 2.
|
5
|
-
- 2.
|
6
|
-
- 2.5.0
|
2
|
+
- 2.4.10
|
3
|
+
- 2.5
|
4
|
+
- 2.6
|
5
|
+
- 2.7
|
7
6
|
- ruby-head
|
8
7
|
|
9
8
|
gemfile:
|
@@ -12,12 +11,6 @@ gemfile:
|
|
12
11
|
services:
|
13
12
|
- mongodb
|
14
13
|
|
15
|
-
before_install:
|
16
|
-
- gem update --system=2.7.8
|
17
|
-
|
18
|
-
before_script:
|
19
|
-
- git submodule update -i
|
20
|
-
|
21
14
|
script: bundle exec rake test
|
22
15
|
|
23
16
|
matrix:
|
data/ChangeLog
CHANGED
data/README.rdoc
CHANGED
@@ -53,6 +53,12 @@ Use _mongo_ type in match.
|
|
53
53
|
capped
|
54
54
|
capped_size 100m
|
55
55
|
|
56
|
+
# Specify date fields in record to use MongoDB's Date object (Optional) default: nil
|
57
|
+
# Supported data types are String/Integer/Float/Fuentd EventTime.
|
58
|
+
# For Integer type, milliseconds epoch and seconds epoch are supported.
|
59
|
+
# eg: updated_at: "2020-02-01T08:22:23.780Z" or updated_at: 1580546457010
|
60
|
+
date_keys updated_at
|
61
|
+
|
56
62
|
# Other buffer configurations here
|
57
63
|
</match>
|
58
64
|
|
@@ -223,7 +229,7 @@ BSON records which include '.' or start with '$' are invalid and they will be st
|
|
223
229
|
...
|
224
230
|
# replace '.' in keys with '__dot__'
|
225
231
|
replace_dot_in_key_with __dot__
|
226
|
-
|
232
|
+
|
227
233
|
# replace '$' in keys with '__dollar__'
|
228
234
|
# Note: This replaces '$' only on first character
|
229
235
|
replace_dollar_in_key_with __dollar__
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.4.0
|
data/fluent-plugin-mongo.gemspec
CHANGED
@@ -10,7 +10,6 @@ Gem::Specification.new do |gem|
|
|
10
10
|
gem.version = File.read("VERSION").strip
|
11
11
|
gem.authors = ["Masahiro Nakagawa"]
|
12
12
|
gem.email = "repeatedly@gmail.com"
|
13
|
-
gem.has_rdoc = false
|
14
13
|
#gem.platform = Gem::Platform::RUBY
|
15
14
|
gem.files = `git ls-files`.split("\n")
|
16
15
|
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
@@ -38,6 +38,10 @@ module Fluent::Plugin
|
|
38
38
|
desc "Replace dollar with specified string"
|
39
39
|
config_param :replace_dollar_in_key_with, :string, default: nil
|
40
40
|
|
41
|
+
# Additional date field to be used to Date object
|
42
|
+
desc "Specify keys to use MongoDB's Date. Supported value types are Integer/Float/EventTime/String"
|
43
|
+
config_param :date_keys, :array, default: nil
|
44
|
+
|
41
45
|
# tag mapping mode
|
42
46
|
desc "Use tag_mapped mode"
|
43
47
|
config_param :tag_mapped, :bool, default: false,
|
@@ -54,6 +58,7 @@ module Fluent::Plugin
|
|
54
58
|
config_param :ssl_verify, :bool, default: false
|
55
59
|
config_param :ssl_ca_cert, :string, default: nil
|
56
60
|
|
61
|
+
|
57
62
|
config_section :buffer do
|
58
63
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
59
64
|
config_set_default :chunk_keys, ['tag']
|
@@ -198,11 +203,40 @@ module Fluent::Plugin
|
|
198
203
|
def collect_records(chunk)
|
199
204
|
records = []
|
200
205
|
time_key = @inject_config.time_key if @inject_config
|
206
|
+
date_keys = @date_keys
|
207
|
+
|
201
208
|
tag = chunk.metadata.tag
|
202
209
|
chunk.msgpack_each {|time, record|
|
203
210
|
record = inject_values_to_record(tag, time, record)
|
204
211
|
# MongoDB uses BSON's Date for time.
|
205
212
|
record[time_key] = Time.at(time || record[time_key]) if time_key
|
213
|
+
|
214
|
+
if date_keys
|
215
|
+
date_keys.each { |date_key|
|
216
|
+
begin
|
217
|
+
date_value = record[date_key]
|
218
|
+
case date_value
|
219
|
+
when Fluent::EventTime
|
220
|
+
record[date_key] = date_value.to_time
|
221
|
+
when Integer
|
222
|
+
record[date_key] = if date_value > 9999999999
|
223
|
+
# epoch with milliseconds: e.g. javascript
|
224
|
+
Time.at(date_value / 1000.0)
|
225
|
+
else
|
226
|
+
# epoch with seconds: e.g. ruby
|
227
|
+
Time.at(date_value)
|
228
|
+
end
|
229
|
+
when Float
|
230
|
+
record[date_key] = Time.at(date_value)
|
231
|
+
else
|
232
|
+
record[date_key] = Time.parse(date_value)
|
233
|
+
end
|
234
|
+
rescue ArgumentError
|
235
|
+
log.warn "Failed to parse '#{date_key}' field. Expected date types are Integer/Float/String/EventTime: #{record[date_key]}"
|
236
|
+
record[date_key] = nil
|
237
|
+
end
|
238
|
+
}
|
239
|
+
end
|
206
240
|
records << record
|
207
241
|
}
|
208
242
|
records
|
@@ -100,7 +100,7 @@ class MongoTailInputTest < Test::Unit::TestCase
|
|
100
100
|
end
|
101
101
|
events = d.events
|
102
102
|
assert_equal "input.mongo", events[0][0]
|
103
|
-
|
103
|
+
assert events[0][1].is_a?(Fluent::EventTime)
|
104
104
|
assert_equal "test", events[0][2]["message"]
|
105
105
|
end
|
106
106
|
|
@@ -138,7 +138,7 @@ class MongoTailInputTest < Test::Unit::TestCase
|
|
138
138
|
events = d.events
|
139
139
|
assert_equal 1, events.size
|
140
140
|
assert_equal "input.mongo.last_id", events[0][0]
|
141
|
-
|
141
|
+
assert events[0][1].is_a?(Fluent::EventTime)
|
142
142
|
assert_equal "can obtain", events[0][2]["message"]
|
143
143
|
end
|
144
144
|
end
|
@@ -315,7 +315,7 @@ class MongoOutputTest < ::Test::Unit::TestCase
|
|
315
315
|
"$foo$bar" => "baz"
|
316
316
|
}
|
317
317
|
],
|
318
|
-
|
318
|
+
})
|
319
319
|
end
|
320
320
|
|
321
321
|
documents = get_documents
|
@@ -388,4 +388,56 @@ class MongoOutputTest < ::Test::Unit::TestCase
|
|
388
388
|
assert authenticate(@client)
|
389
389
|
end
|
390
390
|
end
|
391
|
+
|
392
|
+
sub_test_case 'date_keys' do
|
393
|
+
setup do
|
394
|
+
@updated_at_str = "2020-02-01T08:22:23.780Z"
|
395
|
+
@updated_at_t = Time.parse(@updated_at_str)
|
396
|
+
end
|
397
|
+
|
398
|
+
def emit_date_documents(d)
|
399
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
400
|
+
d.feed(time, {'a' => 1, updated_at: @updated_at_str})
|
401
|
+
d.feed(time, {'a' => 2, updated_at: @updated_at_t.to_f})
|
402
|
+
d.feed(time, {'a' => 3, updated_at: @updated_at_t.to_i})
|
403
|
+
time
|
404
|
+
end
|
405
|
+
|
406
|
+
def emit_invalid_date_documents(d)
|
407
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
408
|
+
d.feed(time, {'a' => 1, updated_at: "Invalid Date String"})
|
409
|
+
time
|
410
|
+
end
|
411
|
+
|
412
|
+
def test_write_with_date_keys
|
413
|
+
d = create_driver(default_config + %[
|
414
|
+
date_keys updated_at
|
415
|
+
time_key created_at
|
416
|
+
])
|
417
|
+
|
418
|
+
d.run(default_tag: 'test') do
|
419
|
+
emit_date_documents(d)
|
420
|
+
end
|
421
|
+
|
422
|
+
actual_documents = get_documents
|
423
|
+
date_key = d.instance.date_keys.first
|
424
|
+
actual_documents.each_with_index { |doc, i|
|
425
|
+
assert_equal(i + 1, doc['a'])
|
426
|
+
assert doc[date_key].is_a?(Time)
|
427
|
+
}
|
428
|
+
end
|
429
|
+
|
430
|
+
def test_write_with_parsed_date_key_invalid_string
|
431
|
+
d = create_driver(default_config + %[
|
432
|
+
date_keys updated_at
|
433
|
+
time_key created_at
|
434
|
+
])
|
435
|
+
|
436
|
+
d.run(default_tag: 'test') do
|
437
|
+
emit_invalid_date_documents(d)
|
438
|
+
end
|
439
|
+
actual_documents = get_documents
|
440
|
+
assert_nil actual_documents.first['updated_at']
|
441
|
+
end
|
442
|
+
end
|
391
443
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mongo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-04-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -159,8 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
159
159
|
- !ruby/object:Gem::Version
|
160
160
|
version: '0'
|
161
161
|
requirements: []
|
162
|
-
|
163
|
-
rubygems_version: 2.6.14.1
|
162
|
+
rubygems_version: 3.0.3
|
164
163
|
signing_key:
|
165
164
|
specification_version: 4
|
166
165
|
summary: MongoDB plugin for Fluentd
|