fluent-plugin-elasticsearch 3.7.0 → 3.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +5 -0
- data/README.ElasticsearchInput.md +1 -1
- data/README.md +5 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_index_template.rb +5 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +11 -3
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +4 -0
- data/test/plugin/test_out_elasticsearch.rb +71 -0
- data/test/plugin/test_out_elasticsearch_dynamic.rb +34 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 66d34ebe18eabf21cd55dbe1c0f3d4f469298e3ada1072849068c1d490e62650
|
4
|
+
data.tar.gz: f91f6d38cc821642ddd6bc2fda232534b14d070fa9694db328bdef68d982d93a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d83715d884fd22a0cb487e0eea16d37679800ce477bfcc888546d1856e48dd8bfa45106826731008b5b3d67cb2408a4d7eca9740a92076ec877a0f999252a512
|
7
|
+
data.tar.gz: d5e476b7ec45883dceeecffeb529d43c1b5a16a9a6c203139ed42891a8e702bc74861edcced7bfd87340e4f3150de0f5a7f7d9ecff8840d1dab9e1348db11601
|
data/History.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
|
+
### 3.7.1
|
5
|
+
- Make conpatible for Fluentd v1.8 (#677)
|
6
|
+
- Handle flatten_hashes in elasticsearch_dynamic (#675)
|
7
|
+
- Handle empty index_date_pattern parameter (#674)
|
8
|
+
|
4
9
|
### 3.7.0
|
5
10
|
- Tweak for cosmetic change (#671)
|
6
11
|
- Fix access to Elasticsearch::Transport::VERSION with explicit top level class path (#670)
|
@@ -284,7 +284,7 @@ Elasticsearch Input plugin and Elasticsearch output plugin can combine to transf
|
|
284
284
|
host transferred-cluster.local
|
285
285
|
port 9200
|
286
286
|
index_name ${$.@metadata._index}
|
287
|
-
|
287
|
+
type_name ${$.@metadata._type} # This parameter will be deprecated due to Removal of mapping types since ES7.
|
288
288
|
id_key ${$.@metadata._id} # This parameter is needed for prevent duplicated records.
|
289
289
|
<buffer tag, $.@metadata._index, $.@metadata._type, $.@metadata._id>
|
290
290
|
@type memory # should use file buffer for preventing chunk lost
|
data/README.md
CHANGED
@@ -434,10 +434,15 @@ for example: <logstash-default-{now/d}-000001>. Overriding this changes the roll
|
|
434
434
|
"now/w{xxxx.ww}" would create weekly rollover indexes instead of daily.
|
435
435
|
|
436
436
|
This setting only takes effect when combined with the [rollover_index](#rollover_index) setting.
|
437
|
+
|
438
|
+
And rollover\_index is also used in Lifecycle Index Management(ILM) feature.
|
437
439
|
```
|
438
440
|
index_date_pattern "now/w{xxxx.ww}" # defaults to "now/d"
|
439
441
|
```
|
440
442
|
|
443
|
+
If empty string(`""`) is specified in `index_date_patter`, index date pattern is not used.
|
444
|
+
Elasticsearch plugin just creates <`index_prefix`-`application_name`-000001> rollover index instead of <`index_prefix`-`application_name`-`{index_date_pattern}`-000001>.
|
445
|
+
|
441
446
|
If [customize_template](#customize_template) is set, then this parameter will be in effect otherwise ignored.
|
442
447
|
|
443
448
|
### deflector_alias
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '3.7.
|
6
|
+
s.version = '3.7.1'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -130,7 +130,11 @@ module Fluent::ElasticsearchIndexTemplate
|
|
130
130
|
def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, host)
|
131
131
|
if rollover_index
|
132
132
|
if !client.indices.exists_alias(:name => deflector_alias_name)
|
133
|
-
|
133
|
+
if index_date_pattern.empty?
|
134
|
+
index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-000001>'
|
135
|
+
else
|
136
|
+
index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
|
137
|
+
end
|
134
138
|
indexcreation(index_name_temp, host)
|
135
139
|
body = {}
|
136
140
|
body = rollover_alias_payload(deflector_alias_name) if enable_ilm
|
@@ -206,7 +206,7 @@ EOC
|
|
206
206
|
end
|
207
207
|
|
208
208
|
@alias_indexes = []
|
209
|
-
if !
|
209
|
+
if !dry_run?
|
210
210
|
if @template_name && @template_file
|
211
211
|
if @rollover_index
|
212
212
|
raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
|
@@ -259,7 +259,7 @@ EOC
|
|
259
259
|
end
|
260
260
|
|
261
261
|
@last_seen_major_version =
|
262
|
-
if @verify_es_version_at_startup && !
|
262
|
+
if @verify_es_version_at_startup && !dry_run?
|
263
263
|
retry_operate(@max_retry_get_es_version) do
|
264
264
|
detect_es_major_version
|
265
265
|
end
|
@@ -278,7 +278,7 @@ EOC
|
|
278
278
|
@type_name = nil
|
279
279
|
end
|
280
280
|
|
281
|
-
if @validate_client_version && !
|
281
|
+
if @validate_client_version && !dry_run?
|
282
282
|
if @last_seen_major_version != client_library_version.to_i
|
283
283
|
raise Fluent::ConfigError, <<-EOC
|
284
284
|
Detected ES #{@last_seen_major_version} but you use ES client #{client_library_version}.
|
@@ -343,6 +343,14 @@ EOC
|
|
343
343
|
end
|
344
344
|
end
|
345
345
|
|
346
|
+
def dry_run?
|
347
|
+
if Fluent::Engine.respond_to?(:dry_run_mode)
|
348
|
+
Fluent::Engine.dry_run_mode
|
349
|
+
elsif Fluent::Engine.respond_to?(:supervisor_mode)
|
350
|
+
Fluent::Engine.supervisor_mode
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
346
354
|
def placeholder?(name, param)
|
347
355
|
begin
|
348
356
|
placeholder_validate!(name, param)
|
@@ -132,6 +132,10 @@ module Fluent::Plugin
|
|
132
132
|
chunk.msgpack_each do |time, record|
|
133
133
|
next unless record.is_a? Hash
|
134
134
|
|
135
|
+
if @flatten_hashes
|
136
|
+
record = flatten_record(record)
|
137
|
+
end
|
138
|
+
|
135
139
|
begin
|
136
140
|
# evaluate all configurations here
|
137
141
|
DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
|
@@ -672,6 +672,77 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
672
672
|
assert_requested(elastic_request)
|
673
673
|
end
|
674
674
|
|
675
|
+
def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
|
676
|
+
cwd = File.dirname(__FILE__)
|
677
|
+
template_file = File.join(cwd, 'test_template.json')
|
678
|
+
|
679
|
+
config = %{
|
680
|
+
host logs.google.com
|
681
|
+
port 777
|
682
|
+
scheme https
|
683
|
+
path /es/
|
684
|
+
user john
|
685
|
+
password doe
|
686
|
+
template_name logstash
|
687
|
+
template_file #{template_file}
|
688
|
+
rollover_index true
|
689
|
+
index_date_pattern ""
|
690
|
+
deflector_alias myapp_deflector
|
691
|
+
enable_ilm true
|
692
|
+
}
|
693
|
+
|
694
|
+
# connection start
|
695
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
696
|
+
with(basic_auth: ['john', 'doe']).
|
697
|
+
to_return(:status => 200, :body => "", :headers => {})
|
698
|
+
# check if template exists
|
699
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
700
|
+
with(basic_auth: ['john', 'doe']).
|
701
|
+
to_return(:status => 404, :body => "", :headers => {})
|
702
|
+
# creation
|
703
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
704
|
+
with(basic_auth: ['john', 'doe']).
|
705
|
+
to_return(:status => 200, :body => "", :headers => {})
|
706
|
+
# check if alias exists
|
707
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
708
|
+
with(basic_auth: ['john', 'doe']).
|
709
|
+
to_return(:status => 404, :body => "", :headers => {})
|
710
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
711
|
+
with(basic_auth: ['john', 'doe']).
|
712
|
+
to_return(status: 404, body: "", headers: {})
|
713
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
714
|
+
with(basic_auth: ['john', 'doe'],
|
715
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
|
716
|
+
to_return(status: 200, body: "", headers: {})
|
717
|
+
# put the alias for the index
|
718
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
|
719
|
+
with(basic_auth: ['john', 'doe']).
|
720
|
+
to_return(:status => 200, :body => "", :headers => {})
|
721
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/_alias/myapp_deflector").
|
722
|
+
with(basic_auth: ['john', 'doe'],
|
723
|
+
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
724
|
+
to_return(:status => 200, :body => "", :headers => {})
|
725
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
726
|
+
with(basic_auth: ['john', 'doe']).
|
727
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
728
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
729
|
+
with(basic_auth: ['john', 'doe']).
|
730
|
+
to_return(:status => 404, :body => "", :headers => {})
|
731
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
732
|
+
with(basic_auth: ['john', 'doe'],
|
733
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
734
|
+
to_return(:status => 200, :body => "", :headers => {})
|
735
|
+
|
736
|
+
driver(config)
|
737
|
+
|
738
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
739
|
+
driver.run(default_tag: 'test') do
|
740
|
+
driver.feed(sample_record)
|
741
|
+
end
|
742
|
+
|
743
|
+
assert_requested(elastic_request)
|
744
|
+
end
|
745
|
+
|
675
746
|
def test_template_create_with_rollover_index_and_custom_ilm
|
676
747
|
cwd = File.dirname(__FILE__)
|
677
748
|
template_file = File.join(cwd, 'test_template.json')
|
@@ -465,6 +465,40 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
465
465
|
assert_equal(2000, total)
|
466
466
|
end
|
467
467
|
|
468
|
+
def test_nested_record_with_flattening_on
|
469
|
+
driver.configure("flatten_hashes true
|
470
|
+
flatten_hashes_separator |")
|
471
|
+
|
472
|
+
original_hash = {"foo" => {"bar" => "baz"}, "people" => [
|
473
|
+
{"age" => "25", "height" => "1ft"},
|
474
|
+
{"age" => "30", "height" => "2ft"}
|
475
|
+
]}
|
476
|
+
|
477
|
+
expected_output = {"foo|bar"=>"baz", "people" => [
|
478
|
+
{"age" => "25", "height" => "1ft"},
|
479
|
+
{"age" => "30", "height" => "2ft"}
|
480
|
+
]}
|
481
|
+
|
482
|
+
stub_elastic
|
483
|
+
driver.run(default_tag: 'test') do
|
484
|
+
driver.feed(original_hash)
|
485
|
+
end
|
486
|
+
assert_equal expected_output, index_cmds[1]
|
487
|
+
end
|
488
|
+
|
489
|
+
def test_nested_record_with_flattening_off
|
490
|
+
# flattening off by default
|
491
|
+
|
492
|
+
original_hash = {"foo" => {"bar" => "baz"}}
|
493
|
+
expected_output = {"foo" => {"bar" => "baz"}}
|
494
|
+
|
495
|
+
stub_elastic
|
496
|
+
driver.run(default_tag: 'test') do
|
497
|
+
driver.feed(original_hash)
|
498
|
+
end
|
499
|
+
assert_equal expected_output, index_cmds[1]
|
500
|
+
end
|
501
|
+
|
468
502
|
def test_makes_bulk_request
|
469
503
|
stub_elastic
|
470
504
|
driver.run(default_tag: 'test') do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.7.
|
4
|
+
version: 3.7.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2019-
|
13
|
+
date: 2019-12-04 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|