fluent-plugin-elasticsearch 1.10.1 → 1.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +3 -0
- data/README.md +9 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +9 -4
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +4 -2
- data/test/plugin/test_out_elasticsearch.rb +55 -1
- data/test/plugin/test_out_elasticsearch_dynamic.rb +55 -6
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7756405c1e59422b26624fbd6e3a0fded30c6d46
|
4
|
+
data.tar.gz: c12bd57fb39982e51e9a5a4dfaa73f3ef5da3e2c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7fb789e36c7a2ad8ae823fa4ed0f7578293bd70426629fccdb09681cc82bbc4e7183439dee046a117bb08ea7de4bcc69eed8052822e5c454e1934218db8d23f2
|
7
|
+
data.tar.gz: ea0983374a67d7f23ad2e5c2a043de495131784b6b583abbd12d483946dc13aa9fc01e9601fabded84ef11404efd23643f280ff9623aafdb4ce5d1a02f8db43a
|
data/History.md
CHANGED
@@ -4,6 +4,9 @@
|
|
4
4
|
- Log ES response errors (#230)
|
5
5
|
- Use latest elasticsearch-ruby (#240)
|
6
6
|
|
7
|
+
### 1.10.2
|
8
|
+
- backport adding `include_timestamp` option (#311)
|
9
|
+
|
7
10
|
### 1.10.1
|
8
11
|
- backport escaping basic authentication user information placeholders (#309)
|
9
12
|
- backport handling dynamic config misconfiguration (#308)
|
data/README.md
CHANGED
@@ -26,6 +26,7 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
|
|
26
26
|
+ [time_precision](#time_precision)
|
27
27
|
+ [time_key](#time_key)
|
28
28
|
+ [time_key_exclude_timestamp](#time_key_exclude_timestamp)
|
29
|
+
+ [include_timestamp](#time_key_exclude_timestamp)
|
29
30
|
+ [utc_index](#utc_index)
|
30
31
|
+ [target_index_key](#target_index_key)
|
31
32
|
+ [target_type_key](#target_type_key)
|
@@ -130,6 +131,14 @@ logstash_format true # defaults to false
|
|
130
131
|
|
131
132
|
This is meant to make writing data into ElasticSearch indices compatible to what [Logstash](https://www.elastic.co/products/logstash) calls them. By doing this, one could take advantage of [Kibana](https://www.elastic.co/products/kibana). See logstash_prefix and logstash_dateformat to customize this index name pattern. The index name will be `#{logstash_prefix}-#{formated_date}`
|
132
133
|
|
134
|
+
### include_timestamp
|
135
|
+
|
136
|
+
```
|
137
|
+
include_timestamp true # defaults to false
|
138
|
+
```
|
139
|
+
|
140
|
+
Adds a `@timestamp` field to the log, following all settings `logstash_format` does, except without the restrictions on `index_name`. This allows one to log to an alias in Elasticsearch and utilize the rollover API.
|
141
|
+
|
133
142
|
### logstash_prefix
|
134
143
|
|
135
144
|
```
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '1.10.
|
6
|
+
s.version = '1.10.2'
|
7
7
|
s.authors = ['diogo', 'pitr']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
|
9
9
|
s.description = %q{ElasticSearch output plugin for Fluent event collector}
|
@@ -28,6 +28,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
|
|
28
28
|
config_param :target_type_key, :string, :default => nil
|
29
29
|
config_param :time_key_format, :string, :default => nil
|
30
30
|
config_param :time_precision, :integer, :default => 0
|
31
|
+
config_param :include_timestamp, :bool, :default => false
|
31
32
|
config_param :logstash_format, :bool, :default => false
|
32
33
|
config_param :logstash_prefix, :string, :default => "logstash"
|
33
34
|
config_param :logstash_prefix_separator, :string, :default => '-'
|
@@ -310,10 +311,8 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
|
|
310
311
|
record = flatten_record(record)
|
311
312
|
end
|
312
313
|
|
313
|
-
|
314
|
-
if
|
315
|
-
target_index = target_index_parent.delete(target_index_child_key)
|
316
|
-
elsif @logstash_format
|
314
|
+
dt = nil
|
315
|
+
if @logstash_format || @include_timestamp
|
317
316
|
if record.has_key?(TIMESTAMP_FIELD)
|
318
317
|
rts = record[TIMESTAMP_FIELD]
|
319
318
|
dt = parse_time(rts, time, tag)
|
@@ -325,6 +324,12 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
|
|
325
324
|
dt = Time.at(time).to_datetime
|
326
325
|
record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)
|
327
326
|
end
|
327
|
+
end
|
328
|
+
|
329
|
+
target_index_parent, target_index_child_key = @target_index_key ? get_parent_of(record, @target_index_key) : nil
|
330
|
+
if target_index_parent && target_index_parent[target_index_child_key]
|
331
|
+
target_index = target_index_parent.delete(target_index_child_key)
|
332
|
+
elsif @logstash_format
|
328
333
|
dt = dt.new_offset(0) if @utc_index
|
329
334
|
target_index = "#{@logstash_prefix}#{@logstash_prefix_separator}#{dt.strftime(@logstash_dateformat)}"
|
330
335
|
else
|
@@ -7,7 +7,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
|
|
7
7
|
|
8
8
|
config_param :delimiter, :string, :default => "."
|
9
9
|
|
10
|
-
DYNAMIC_PARAM_NAMES = %W[hosts host port logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
|
10
|
+
DYNAMIC_PARAM_NAMES = %W[hosts host port include_timestamp logstash_format logstash_prefix logstash_dateformat time_key utc_index index_name tag_key type_name id_key parent_key routing_key write_operation]
|
11
11
|
DYNAMIC_PARAM_SYMBOLS = DYNAMIC_PARAM_NAMES.map { |n| "@#{n}".to_sym }
|
12
12
|
|
13
13
|
attr_reader :dynamic_config
|
@@ -139,7 +139,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
|
|
139
139
|
next
|
140
140
|
end
|
141
141
|
|
142
|
-
if eval_or_val(dynamic_conf['logstash_format'])
|
142
|
+
if eval_or_val(dynamic_conf['logstash_format']) || eval_or_val(dynamic_conf['include_timestamp'])
|
143
143
|
if record.has_key?("@timestamp")
|
144
144
|
time = Time.parse record["@timestamp"]
|
145
145
|
elsif record.has_key?(dynamic_conf['time_key'])
|
@@ -148,7 +148,9 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
|
|
148
148
|
else
|
149
149
|
record.merge!({"@timestamp" => Time.at(time).to_datetime.to_s})
|
150
150
|
end
|
151
|
+
end
|
151
152
|
|
153
|
+
if eval_or_val(dynamic_conf['logstash_format'])
|
152
154
|
if eval_or_val(dynamic_conf['utc_index'])
|
153
155
|
target_index = "#{dynamic_conf['logstash_prefix']}-#{Time.at(time).getutc.strftime("#{dynamic_conf['logstash_dateformat']}")}"
|
154
156
|
else
|
@@ -755,7 +755,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
755
755
|
assert_nil(index_cmds[1]['@timestamp'])
|
756
756
|
end
|
757
757
|
|
758
|
-
def
|
758
|
+
def test_adds_timestamp_when_logstash
|
759
759
|
driver.configure("logstash_format true\n")
|
760
760
|
stub_elastic_ping
|
761
761
|
stub_elastic
|
@@ -766,6 +766,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
766
766
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
767
767
|
end
|
768
768
|
|
769
|
+
def test_adds_timestamp_when_include_timestamp
|
770
|
+
driver.configure("include_timestamp true\n")
|
771
|
+
stub_elastic_ping
|
772
|
+
stub_elastic
|
773
|
+
ts = DateTime.now
|
774
|
+
time = ts.to_time
|
775
|
+
driver.emit(sample_record, time)
|
776
|
+
driver.run
|
777
|
+
tf = "%Y-%m-%dT%H:%M:%S%:z"
|
778
|
+
timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
|
779
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
780
|
+
assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
|
781
|
+
end
|
782
|
+
|
769
783
|
def test_uses_custom_timestamp_when_included_in_record
|
770
784
|
driver.configure("logstash_format true\n")
|
771
785
|
stub_elastic_ping
|
@@ -777,6 +791,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
777
791
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
778
792
|
end
|
779
793
|
|
794
|
+
def test_uses_custom_timestamp_when_included_in_record_without_logstash
|
795
|
+
driver.configure("include_timestamp true\n")
|
796
|
+
stub_elastic_ping
|
797
|
+
stub_elastic
|
798
|
+
ts = DateTime.new(2001,2,3).to_s
|
799
|
+
driver.emit(sample_record.merge!('@timestamp' => ts))
|
800
|
+
driver.run
|
801
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
802
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
803
|
+
end
|
804
|
+
|
780
805
|
def test_uses_custom_time_key
|
781
806
|
driver.configure("logstash_format true
|
782
807
|
time_key vtm\n")
|
@@ -803,6 +828,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
803
828
|
assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
|
804
829
|
end
|
805
830
|
|
831
|
+
def test_uses_custom_time_key_with_format_without_logstash
|
832
|
+
driver.configure("include_timestamp true
|
833
|
+
index_name test
|
834
|
+
time_key_format %Y-%m-%d %H:%M:%S.%N%z
|
835
|
+
time_key vtm\n")
|
836
|
+
stub_elastic_ping
|
837
|
+
stub_elastic
|
838
|
+
ts = "2001-02-03 13:14:01.673+02:00"
|
839
|
+
driver.emit(sample_record.merge!('vtm' => ts))
|
840
|
+
driver.run
|
841
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
842
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
843
|
+
assert_equal("test", index_cmds[0]['index']['_index'])
|
844
|
+
end
|
845
|
+
|
806
846
|
def test_uses_custom_time_key_exclude_timekey
|
807
847
|
driver.configure("logstash_format true
|
808
848
|
time_key vtm
|
@@ -828,6 +868,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
828
868
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
829
869
|
end
|
830
870
|
|
871
|
+
def test_uses_custom_time_key_format_without_logstash
|
872
|
+
driver.configure("include_timestamp true
|
873
|
+
index_name test
|
874
|
+
time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
|
875
|
+
stub_elastic_ping
|
876
|
+
stub_elastic
|
877
|
+
ts = "2001-02-03T13:14:01.673+02:00"
|
878
|
+
driver.emit(sample_record.merge!('@timestamp' => ts))
|
879
|
+
driver.run
|
880
|
+
assert_equal("test", index_cmds[0]['index']['_index'])
|
881
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
882
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
883
|
+
end
|
884
|
+
|
831
885
|
data(:default => nil,
|
832
886
|
:custom_tag => 'es_plugin.output.time.error')
|
833
887
|
def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
|
@@ -232,7 +232,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
232
232
|
assert_equal('fluentd', index_cmds.first['index']['_type'])
|
233
233
|
end
|
234
234
|
|
235
|
-
def
|
235
|
+
def test_writes_to_specified_index
|
236
236
|
driver.configure("index_name myindex\n")
|
237
237
|
stub_elastic_ping
|
238
238
|
stub_elastic
|
@@ -241,7 +241,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
241
241
|
assert_equal('myindex', index_cmds.first['index']['_index'])
|
242
242
|
end
|
243
243
|
|
244
|
-
def
|
244
|
+
def test_writes_to_specified_index_uppercase
|
245
245
|
driver.configure("index_name MyIndex\n")
|
246
246
|
stub_elastic_ping
|
247
247
|
stub_elastic
|
@@ -250,7 +250,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
250
250
|
assert_equal('myindex', index_cmds.first['index']['_index'])
|
251
251
|
end
|
252
252
|
|
253
|
-
def
|
253
|
+
def test_writes_to_specified_type
|
254
254
|
driver.configure("type_name mytype\n")
|
255
255
|
stub_elastic_ping
|
256
256
|
stub_elastic
|
@@ -259,7 +259,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
259
259
|
assert_equal('mytype', index_cmds.first['index']['_type'])
|
260
260
|
end
|
261
261
|
|
262
|
-
def
|
262
|
+
def test_writes_to_specified_host
|
263
263
|
driver.configure("host 192.168.33.50\n")
|
264
264
|
stub_elastic_ping("http://192.168.33.50:9200")
|
265
265
|
elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
|
@@ -268,7 +268,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
268
268
|
assert_requested(elastic_request)
|
269
269
|
end
|
270
270
|
|
271
|
-
def
|
271
|
+
def test_writes_to_specified_port
|
272
272
|
driver.configure("port 9201\n")
|
273
273
|
stub_elastic_ping("http://localhost:9201")
|
274
274
|
elastic_request = stub_elastic("http://localhost:9201/_bulk")
|
@@ -419,6 +419,17 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
419
419
|
end
|
420
420
|
|
421
421
|
def test_uses_custom_timestamp_when_included_in_record
|
422
|
+
driver.configure("include_timestamp true\n")
|
423
|
+
stub_elastic_ping
|
424
|
+
stub_elastic
|
425
|
+
ts = DateTime.new(2001,2,3).to_s
|
426
|
+
driver.emit(sample_record.merge!('@timestamp' => ts))
|
427
|
+
driver.run
|
428
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
429
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
430
|
+
end
|
431
|
+
|
432
|
+
def test_uses_custom_timestamp_when_included_in_record_logstash
|
422
433
|
driver.configure("logstash_format true\n")
|
423
434
|
stub_elastic_ping
|
424
435
|
stub_elastic
|
@@ -429,7 +440,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
429
440
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
430
441
|
end
|
431
442
|
|
432
|
-
def
|
443
|
+
def test_uses_custom_time_key_logstash
|
433
444
|
driver.configure("logstash_format true
|
434
445
|
time_key vtm\n")
|
435
446
|
stub_elastic_ping
|
@@ -441,7 +452,45 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
441
452
|
assert_equal(index_cmds[1]['@timestamp'], ts)
|
442
453
|
end
|
443
454
|
|
455
|
+
def test_uses_custom_time_key_timestamp
|
456
|
+
driver.configure("include_timestamp true
|
457
|
+
time_key vtm\n")
|
458
|
+
stub_elastic_ping
|
459
|
+
stub_elastic
|
460
|
+
ts = DateTime.new(2001,2,3).to_s
|
461
|
+
driver.emit(sample_record.merge!('vtm' => ts))
|
462
|
+
driver.run
|
463
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
464
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
465
|
+
end
|
466
|
+
|
467
|
+
def test_uses_custom_time_key_timestamp_custom_index
|
468
|
+
driver.configure("include_timestamp true
|
469
|
+
index_name test
|
470
|
+
time_key vtm\n")
|
471
|
+
stub_elastic_ping
|
472
|
+
stub_elastic
|
473
|
+
ts = DateTime.new(2001,2,3).to_s
|
474
|
+
driver.emit(sample_record.merge!('vtm' => ts))
|
475
|
+
driver.run
|
476
|
+
assert(index_cmds[1].has_key? '@timestamp')
|
477
|
+
assert_equal(index_cmds[1]['@timestamp'], ts)
|
478
|
+
assert_equal('test', index_cmds.first['index']['_index'])
|
479
|
+
end
|
480
|
+
|
444
481
|
def test_uses_custom_time_key_exclude_timestamp
|
482
|
+
driver.configure("include_timestamp true
|
483
|
+
time_key vtm
|
484
|
+
time_key_exclude_timestamp true\n")
|
485
|
+
stub_elastic_ping
|
486
|
+
stub_elastic
|
487
|
+
ts = DateTime.new(2001,2,3).to_s
|
488
|
+
driver.emit(sample_record.merge!('vtm' => ts))
|
489
|
+
driver.run
|
490
|
+
assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be missing')
|
491
|
+
end
|
492
|
+
|
493
|
+
def test_uses_custom_time_key_exclude_timestamp_logstash
|
445
494
|
driver.configure("logstash_format true
|
446
495
|
time_key vtm
|
447
496
|
time_key_exclude_timestamp true\n")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.10.
|
4
|
+
version: 1.10.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-10-
|
12
|
+
date: 2017-10-25 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|