fluent-plugin-elasticsearch 2.0.0.rc.6 → 2.0.0.rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +3 -0
- data/README.md +9 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +9 -4
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +4 -2
- data/test/plugin/test_out_elasticsearch.rb +59 -1
- data/test/plugin/test_out_elasticsearch_dynamic.rb +59 -6
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f225504787161485e7c1623d16d248cfddbccb0e
|
4
|
+
data.tar.gz: 208b3f560dfb6f76c8154abd5aa935cd488330ef
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 643760687fe81204c27d67b31a30538ec8d49baca9bc96797c836c8029fa06ca2d4fb0610f326ccf82a1ae047ad92429b5ef3a8319ce05f93a8dd2105136993b
|
7
|
+
data.tar.gz: 0bfce17925d11c35c658c3439cf3888358b2e3699e91351c149871357916b3165c45a0750203585fa66a4175b288d0008b40b4c11530e6b3cdb0509ca23bde0f
|
data/History.md
CHANGED
data/README.md
CHANGED
@@ -28,6 +28,7 @@ Current maintainers: @cosmo0920
|
|
28
28
|
+ [time_precision](#time_precision)
|
29
29
|
+ [time_key](#time_key)
|
30
30
|
+ [time_key_exclude_timestamp](#time_key_exclude_timestamp)
|
31
|
+
+ [include_timestamp](#time_key_exclude_timestamp)
|
31
32
|
+ [utc_index](#utc_index)
|
32
33
|
+ [target_index_key](#target_index_key)
|
33
34
|
+ [target_type_key](#target_type_key)
|
@@ -147,6 +148,14 @@ This is meant to make writing data into ElasticSearch indices compatible to what
|
|
147
148
|
|
148
149
|
:warning: Setting this option to `true` will ignore the `index_name` setting. The default index name prefix is `logstash-`.
|
149
150
|
|
151
|
+
### include_timestamp
|
152
|
+
|
153
|
+
```
|
154
|
+
include_timestamp true # defaults to false
|
155
|
+
```
|
156
|
+
|
157
|
+
Adds a `@timestamp` field to the log, following all settings `logstash_format` does, except without the restrictions on `index_name`. This allows one to log to an alias in Elasticsearch and utilize the rollover API.
|
158
|
+
|
150
159
|
### logstash_prefix
|
151
160
|
|
152
161
|
```
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '2.0.0.rc.
|
6
|
+
s.version = '2.0.0.rc.7'
|
7
7
|
s.authors = ['diogo', 'pitr']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
|
9
9
|
s.description = %q{ElasticSearch output plugin for Fluent event collector}
|
@@ -33,6 +33,7 @@ module Fluent::Plugin
|
|
33
33
|
config_param :target_type_key, :string, :default => nil
|
34
34
|
config_param :time_key_format, :string, :default => nil
|
35
35
|
config_param :time_precision, :integer, :default => 9
|
36
|
+
config_param :include_timestamp, :bool, :default => false
|
36
37
|
config_param :logstash_format, :bool, :default => false
|
37
38
|
config_param :logstash_prefix, :string, :default => "logstash"
|
38
39
|
config_param :logstash_prefix_separator, :string, :default => '-'
|
@@ -339,10 +340,8 @@ module Fluent::Plugin
|
|
339
340
|
record = flatten_record(record)
|
340
341
|
end
|
341
342
|
|
342
|
-
|
343
|
-
if
|
344
|
-
target_index = target_index_parent.delete(target_index_child_key)
|
345
|
-
elsif @logstash_format
|
343
|
+
dt = nil
|
344
|
+
if @logstash_format || @include_timestamp
|
346
345
|
if record.has_key?(TIMESTAMP_FIELD)
|
347
346
|
rts = record[TIMESTAMP_FIELD]
|
348
347
|
dt = parse_time(rts, time, tag)
|
@@ -354,6 +353,12 @@ module Fluent::Plugin
|
|
354
353
|
dt = Time.at(time).to_datetime
|
355
354
|
record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
|
356
355
|
end
|
356
|
+
end
|
357
|
+
|
358
|
+
target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
|
359
|
+
if target_index_parent && target_index_parent[target_index_child_key]
|
360
|
+
target_index = target_index_parent.delete(target_index_child_key)
|
361
|
+
elsif @logstash_format
|
357
362
|
dt = dt.new_offset(0) if @utc_index
|
358
363
|
target_index = "#{logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
|
359
364
|
else
|
@@ -10,7 +10,7 @@ module Fluent::Plugin
|
|
10
10
|
|
11
11
|
config_param :delimiter, :string, :default => "."
|
12
12
|
|
13
|
-
DYNAMIC_PARAM_NAMES = %W[hosts host port logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
|
13
|
+
DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
|
14
14
|
DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
|
15
15
|
|
16
16
|
attr_reader :dynamic_config
|
@@ -148,7 +148,7 @@ module Fluent::Plugin
|
|
148
148
|
next
|
149
149
|
end
|
150
150
|
|
151
|
-
if eval_or_val(dynamic_conf['logstash_format'])
|
151
|
+
if eval_or_val(dynamic_conf['logstash_format']) || eval_or_val(dynamic_conf['include_timestamp'])
|
152
152
|
if record.has_key?("@timestamp")
|
153
153
|
time = Time.parse record["@timestamp"]
|
154
154
|
elsif record.has_key?(dynamic_conf['time_key'])
|
@@ -157,7 +157,9 @@ module Fluent::Plugin
|
|
157
157
|
else
|
158
158
|
record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
|
159
159
|
end
|
160
|
+
end
|
160
161
|
|
162
|
+
if eval_or_val(dynamic_conf['logstash_format'])
|
161
163
|
if eval_or_val(dynamic_conf['utc_index'])
|
162
164
|
target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
|
163
165
|
else
|
@@ -940,7 +940,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
940
940
|
assert_nil(index_cmds[1]['@timestamp'])
|
941
941
|
end
|
942
942
|
|
943
|
-
def
|
943
|
+
def test_adds_timestamp_when_logstash
|
944
944
|
driver.configure("logstash_format true\n")
|
945
945
|
stub_elastic_ping
|
946
946
|
stub_elastic
|
@@ -955,6 +955,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
955
955
|
assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
|
956
956
|
end
|
957
957
|
|
958
|
+
def test_adds_timestamp_when_include_timestamp
|
959
|
+
driver.configure("include_timestamp true\n")
|
960
|
+
stub_elastic_ping
|
961
|
+
stub_elastic
|
962
|
+
ts = DateTime.now
|
963
|
+
time = Fluent::EventTime.from_time(ts.to_time)
|
964
|
+
driver.run(default_tag: 'test') do
|
965
|
+
driver.feed(time, sample_record)
|
966
|
+
end
|
967
|
+
tf = "%Y-%m-%dT%H:%M:%S%:z"
|
968
|
+
timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
|
969
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
970
|
+
assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
|
971
|
+
end
|
972
|
+
|
958
973
|
def test_uses_custom_timestamp_when_included_in_record
|
959
974
|
driver.configure("logstash_format true\n")
|
960
975
|
stub_elastic_ping
|
@@ -967,6 +982,18 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
967
982
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
968
983
|
end
|
969
984
|
|
985
|
+
def test_uses_custom_timestamp_when_included_in_record_without_logstash
|
986
|
+
driver.configure("include_timestamp true\n")
|
987
|
+
stub_elastic_ping
|
988
|
+
stub_elastic
|
989
|
+
ts = DateTime.new(2001,2,3).to_s
|
990
|
+
driver.run(default_tag: 'test') do
|
991
|
+
driver.feed(sample_record.merge!('@timestamp' => ts))
|
992
|
+
end
|
993
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
994
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
995
|
+
end
|
996
|
+
|
970
997
|
def test_uses_custom_time_key
|
971
998
|
driver.configure("logstash_format true
|
972
999
|
time_key vtm\n")
|
@@ -995,6 +1022,22 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
995
1022
|
assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
|
996
1023
|
end
|
997
1024
|
|
1025
|
+
def test_uses_custom_time_key_with_format_without_logstash
|
1026
|
+
driver.configure("include_timestamp true
|
1027
|
+
index_name test
|
1028
|
+
time_key_format %Y-%m-%d %H:%M:%S.%N%z
|
1029
|
+
time_key vtm\n")
|
1030
|
+
stub_elastic_ping
|
1031
|
+
stub_elastic
|
1032
|
+
ts = "2001-02-03 13:14:01.673+02:00"
|
1033
|
+
driver.run(default_tag: 'test') do
|
1034
|
+
driver.feed(sample_record.merge!('vtm' => ts))
|
1035
|
+
end
|
1036
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
1037
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
1038
|
+
assert_equal("test", index_cmds[0]['index']['_index'])
|
1039
|
+
end
|
1040
|
+
|
998
1041
|
def test_uses_custom_time_key_exclude_timekey
|
999
1042
|
driver.configure("logstash_format true
|
1000
1043
|
time_key vtm
|
@@ -1022,6 +1065,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1022
1065
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
1023
1066
|
end
|
1024
1067
|
|
1068
|
+
def test_uses_custom_time_key_format_without_logstash
|
1069
|
+
driver.configure("include_timestamp true
|
1070
|
+
index_name test
|
1071
|
+
time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
|
1072
|
+
stub_elastic_ping
|
1073
|
+
stub_elastic
|
1074
|
+
ts = "2001-02-03T13:14:01.673+02:00"
|
1075
|
+
driver.run(default_tag: 'test') do
|
1076
|
+
driver.feed(sample_record.merge!('@timestamp' => ts))
|
1077
|
+
end
|
1078
|
+
assert_equal("test", index_cmds[0]['index']['_index'])
|
1079
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
1080
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
1081
|
+
end
|
1082
|
+
|
1025
1083
|
data(:default => nil,
|
1026
1084
|
:custom_tag => 'es_plugin.output.time.error')
|
1027
1085
|
def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
|
@@ -236,7 +236,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
236
236
|
assert_equal('fluentd', index_cmds.first['index']['_type'])
|
237
237
|
end
|
238
238
|
|
239
|
-
def
|
239
|
+
def test_writes_to_specified_index
|
240
240
|
driver.configure("index_name myindex\n")
|
241
241
|
stub_elastic_ping
|
242
242
|
stub_elastic
|
@@ -246,7 +246,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
246
246
|
assert_equal('myindex', index_cmds.first['index']['_index'])
|
247
247
|
end
|
248
248
|
|
249
|
-
def
|
249
|
+
def test_writes_to_specified_index_uppercase
|
250
250
|
driver.configure("index_name MyIndex\n")
|
251
251
|
stub_elastic_ping
|
252
252
|
stub_elastic
|
@@ -256,7 +256,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
256
256
|
assert_equal('myindex', index_cmds.first['index']['_index'])
|
257
257
|
end
|
258
258
|
|
259
|
-
def
|
259
|
+
def test_writes_to_specified_type
|
260
260
|
driver.configure("type_name mytype\n")
|
261
261
|
stub_elastic_ping
|
262
262
|
stub_elastic
|
@@ -266,7 +266,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
266
266
|
assert_equal('mytype', index_cmds.first['index']['_type'])
|
267
267
|
end
|
268
268
|
|
269
|
-
def
|
269
|
+
def test_writes_to_specified_host
|
270
270
|
driver.configure("host 192.168.33.50\n")
|
271
271
|
stub_elastic_ping("http://192.168.33.50:9200")
|
272
272
|
elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
|
@@ -276,7 +276,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
276
276
|
assert_requested(elastic_request)
|
277
277
|
end
|
278
278
|
|
279
|
-
def
|
279
|
+
def test_writes_to_specified_port
|
280
280
|
driver.configure("port 9201\n")
|
281
281
|
stub_elastic_ping("http://localhost:9201")
|
282
282
|
elastic_request = stub_elastic("http://localhost:9201/_bulk")
|
@@ -438,6 +438,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
438
438
|
end
|
439
439
|
|
440
440
|
def test_uses_custom_timestamp_when_included_in_record
|
441
|
+
driver.configure("include_timestamp true\n")
|
442
|
+
stub_elastic_ping
|
443
|
+
stub_elastic
|
444
|
+
ts = DateTime.new(2001,2,3).to_s
|
445
|
+
driver.run(default_tag: 'test') do
|
446
|
+
driver.feed(sample_record.merge!('@timestamp' => ts))
|
447
|
+
end
|
448
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
449
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
450
|
+
end
|
451
|
+
|
452
|
+
def test_uses_custom_timestamp_when_included_in_record_logstash
|
441
453
|
driver.configure("logstash_format true\n")
|
442
454
|
stub_elastic_ping
|
443
455
|
stub_elastic
|
@@ -449,7 +461,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
449
461
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
450
462
|
end
|
451
463
|
|
452
|
-
def
|
464
|
+
def test_uses_custom_time_key_logstash
|
453
465
|
driver.configure("logstash_format true
|
454
466
|
time_key vtm\n")
|
455
467
|
stub_elastic_ping
|
@@ -462,7 +474,48 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
462
474
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
463
475
|
end
|
464
476
|
|
477
|
+
def test_uses_custom_time_key_timestamp
|
478
|
+
driver.configure("include_timestamp true
|
479
|
+
time_key vtm\n")
|
480
|
+
stub_elastic_ping
|
481
|
+
stub_elastic
|
482
|
+
ts = DateTime.new(2001,2,3).to_s
|
483
|
+
driver.run(default_tag: 'test') do
|
484
|
+
driver.feed(sample_record.merge!('vtm' => ts))
|
485
|
+
end
|
486
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
487
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
488
|
+
end
|
489
|
+
|
490
|
+
def test_uses_custom_time_key_timestamp_custom_index
|
491
|
+
driver.configure("include_timestamp true
|
492
|
+
index_name test
|
493
|
+
time_key vtm\n")
|
494
|
+
stub_elastic_ping
|
495
|
+
stub_elastic
|
496
|
+
ts = DateTime.new(2001,2,3).to_s
|
497
|
+
driver.run(default_tag: 'test') do
|
498
|
+
driver.feed(sample_record.merge!('vtm' => ts))
|
499
|
+
end
|
500
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
501
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
502
|
+
assert_equal('test', index_cmds.first['index']['_index'])
|
503
|
+
end
|
504
|
+
|
465
505
|
def test_uses_custom_time_key_exclude_timestamp
|
506
|
+
driver.configure("include_timestamp true
|
507
|
+
time_key vtm
|
508
|
+
time_key_exclude_timestamp true\n")
|
509
|
+
stub_elastic_ping
|
510
|
+
stub_elastic
|
511
|
+
ts = DateTime.new(2001,2,3).to_s
|
512
|
+
driver.run(default_tag: 'test') do
|
513
|
+
driver.feed(sample_record.merge!('vtm' => ts))
|
514
|
+
end
|
515
|
+
assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be missing')
|
516
|
+
end
|
517
|
+
|
518
|
+
def test_uses_custom_time_key_exclude_timestamp_logstash
|
466
519
|
driver.configure("logstash_format true
|
467
520
|
time_key vtm
|
468
521
|
time_key_exclude_timestamp true\n")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.0.rc.
|
4
|
+
version: 2.0.0.rc.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-10-
|
12
|
+
date: 2017-10-25 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|