fluent-plugin-elasticsearch 5.1.4 → 5.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/History.md +13 -0
- data/README.md +16 -1
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +30 -0
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +8 -4
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +2 -1
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +9 -8
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +59 -29
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
- data/test/plugin/test_elasticsearch_fallback_selector.rb +15 -7
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +47 -17
- data/test/plugin/test_in_elasticsearch.rb +10 -2
- data/test/plugin/test_out_elasticsearch.rb +281 -122
- data/test/plugin/test_out_elasticsearch_data_stream.rb +266 -35
- data/test/plugin/test_out_elasticsearch_dynamic.rb +57 -20
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 425652fd5cb31671a87378fdaaf8fa911aa6bda0743fe44c53d7631d5018aebb
|
4
|
+
data.tar.gz: d3fb4697a32f8448fdd9995adeef50a96839eb15b10f984a03618044c7a9ff35
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d8b6b4e919d79de0153707bfb84033ea403c0a88f7d76a7f1d986a45e67a0fec340762bd0a5c8776e2f6dff18b267550623fe516018a634fa3c56b9f63a3937c
|
7
|
+
data.tar.gz: 40b798377a202b4e7c0c0e6f7e84910cf7574711a83a0919c274ff5b8b9eb0dd5b90bd3e5c682ba2be9d684384c29bbbfe67d62d9307e8d0df050ee92d460c0b
|
data/Gemfile
CHANGED
@@ -7,5 +7,5 @@ gem 'simplecov', require: false
|
|
7
7
|
gem 'coveralls', ">= 0.8.0", require: false
|
8
8
|
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
9
|
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
|
-
gem "elasticsearch-xpack"
|
10
|
+
gem "elasticsearch-xpack" if ENV["USE_XPACK"]
|
11
11
|
gem "oj"
|
data/History.md
CHANGED
@@ -2,6 +2,19 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 5.2.1
|
6
|
+
- respect include\_tag\_key and tag\_key setting when using data streams (#936)
|
7
|
+
- Handle unsupported version error (#956)
|
8
|
+
- Display deprecated warning on ES dynamic plugin (#955)
|
9
|
+
|
10
|
+
### 5.2.0
|
11
|
+
- Migrate to handle Elasticsearch 8 (#949)
|
12
|
+
|
13
|
+
### 5.1.5
|
14
|
+
- Make retryable DataStreams creation at configure phase (#943)
|
15
|
+
- Handle @hosts parameter on data_stream plugin (#942)
|
16
|
+
- allow specifying custom ILM policies for data streams (#933)
|
17
|
+
|
5
18
|
### 5.1.4
|
6
19
|
- Handle ES8 or above more strictly (#931)
|
7
20
|
- fixing double "\_policy" in index lifecycle management policy for elasticsearch\_data\_stream output (#930)
|
data/README.md
CHANGED
@@ -1521,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
|
|
1521
1521
|
data_stream_name test
|
1522
1522
|
```
|
1523
1523
|
|
1524
|
-
When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
|
1524
|
+
When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name` or `data_stream_ilm_policy`, ILM default policy is set to the specified data stream.
|
1525
1525
|
Then, the matching index template is also created automatically.
|
1526
1526
|
|
1527
1527
|
### data_stream_name
|
@@ -1545,6 +1545,21 @@ There are some limitations about naming rule.
|
|
1545
1545
|
|
1546
1546
|
In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).
|
1547
1547
|
|
1548
|
+
|
1549
|
+
### data_stream_ilm_policy
|
1550
|
+
|
1551
|
+
You can specify the ILM policy contents as hash. If not present, it will apply the ILM default policy.
|
1552
|
+
|
1553
|
+
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
1554
|
+
|
1555
|
+
### data_stream_ilm_policy_overwrite
|
1556
|
+
|
1557
|
+
Specify whether the data stream ILM policy should be overwritten.
|
1558
|
+
|
1559
|
+
Default value is `false`.
|
1560
|
+
|
1561
|
+
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
1562
|
+
|
1548
1563
|
## Troubleshooting
|
1549
1564
|
|
1550
1565
|
See [Troubleshooting document](README.Troubleshooting.md)
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '5.1
|
6
|
+
s.version = '5.2.1'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -0,0 +1,30 @@
|
|
1
|
+
begin
|
2
|
+
require 'elastic/transport'
|
3
|
+
::TRANSPORT_CLASS = Elastic::Transport
|
4
|
+
rescue LoadError
|
5
|
+
end
|
6
|
+
begin
|
7
|
+
require 'elasticsearch/transport'
|
8
|
+
::TRANSPORT_CLASS = Elasticsearch::Transport
|
9
|
+
rescue LoadError
|
10
|
+
end
|
11
|
+
if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("8.0.0")
|
12
|
+
begin
|
13
|
+
require 'elasticsearch/xpack'
|
14
|
+
rescue LoadError
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
begin
|
19
|
+
require 'elastic/transport/transport/connections/selector'
|
20
|
+
::SELECTOR_CLASS = Elastic::Transport::Transport::Connections::Selector
|
21
|
+
rescue LoadError
|
22
|
+
end
|
23
|
+
begin
|
24
|
+
require 'elasticsearch/transport/transport/connections/selector'
|
25
|
+
::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
|
26
|
+
rescue LoadError
|
27
|
+
end
|
28
|
+
unless defined?(Elasticsearch::UnsupportedProductError)
|
29
|
+
class Elasticsearch::UnsupportedProductError < StandardError; end
|
30
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
|
-
|
1
|
+
require_relative 'elasticsearch_compat'
|
2
2
|
|
3
3
|
class Fluent::Plugin::ElasticseatchFallbackSelector
|
4
|
-
include
|
4
|
+
include SELECTOR_CLASS::Base
|
5
5
|
|
6
6
|
def select(options={})
|
7
7
|
connections.first
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require_relative 'elasticsearch_compat'
|
2
|
+
|
1
3
|
module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
2
4
|
ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
|
3
5
|
|
@@ -21,7 +23,7 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
21
23
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
|
22
24
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
|
23
25
|
|
24
|
-
rescue
|
26
|
+
rescue ::TRANSPORT_CLASS::Transport::Error => e
|
25
27
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
|
26
28
|
end
|
27
29
|
end
|
@@ -43,12 +45,20 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
43
45
|
end
|
44
46
|
|
45
47
|
def get_ilm_policy
|
46
|
-
|
48
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
49
|
+
client.ilm.get_policy
|
50
|
+
else
|
51
|
+
client.enrich.get_policy
|
52
|
+
end
|
47
53
|
end
|
48
54
|
|
49
55
|
def ilm_policy_exists?(policy_id)
|
50
56
|
begin
|
51
|
-
|
57
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
58
|
+
client.ilm.get_policy(policy_id: policy_id)
|
59
|
+
else
|
60
|
+
client.enrich.get_policy(name: policy_id)
|
61
|
+
end
|
52
62
|
true
|
53
63
|
rescue
|
54
64
|
false
|
@@ -57,7 +67,11 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
57
67
|
|
58
68
|
def ilm_policy_put(policy_id, policy)
|
59
69
|
log.info("Installing ILM policy: #{policy}")
|
60
|
-
|
70
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
71
|
+
client.ilm.put_policy(policy_id: policy_id, body: policy)
|
72
|
+
else
|
73
|
+
client.enrich.put_policy(name: policy_id, body: policy)
|
74
|
+
end
|
61
75
|
end
|
62
76
|
|
63
77
|
def default_policy_payload
|
@@ -28,12 +28,16 @@ module Fluent::ElasticsearchIndexTemplate
|
|
28
28
|
client(host).indices.get_index_template(:name => name)
|
29
29
|
end
|
30
30
|
return true
|
31
|
-
rescue
|
31
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound
|
32
32
|
return false
|
33
33
|
end
|
34
34
|
|
35
35
|
def host_unreachable_exceptions
|
36
|
-
if Gem::Version.new(::
|
36
|
+
if Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
37
|
+
# elasticsearch-ruby 8.0.0's elastic-transport uses
|
38
|
+
# direct callable #host_unreachable_exceptions again.
|
39
|
+
client.transport.host_unreachable_exceptions
|
40
|
+
elsif Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
37
41
|
# elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
|
38
42
|
# Elasticsearch class on Transport.
|
39
43
|
# This is why #host_unreachable_exceptions is not callable directly
|
@@ -47,7 +51,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
47
51
|
def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
|
48
52
|
return unless block_given?
|
49
53
|
retries = 0
|
50
|
-
transport_errors =
|
54
|
+
transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map{ |c| TRANSPORT_CLASS::Transport::Errors.const_get c } if catch_trasport_exceptions
|
51
55
|
begin
|
52
56
|
yield
|
53
57
|
rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
|
@@ -78,7 +82,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
78
82
|
|
79
83
|
def indexcreation(index_name, host = nil)
|
80
84
|
client(host).indices.create(:index => index_name)
|
81
|
-
rescue
|
85
|
+
rescue TRANSPORT_CLASS::Transport::Error => e
|
82
86
|
if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
|
83
87
|
log.debug("Index #{index_name} already exists")
|
84
88
|
else
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'elasticsearch'
|
2
|
+
require_relative 'elasticsearch_compat'
|
2
3
|
|
3
|
-
class Fluent::Plugin::ElasticsearchSimpleSniffer <
|
4
|
+
class Fluent::Plugin::ElasticsearchSimpleSniffer < TRANSPORT_CLASS::Transport::Sniffer
|
4
5
|
|
5
6
|
def hosts
|
6
7
|
@transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
|
@@ -3,6 +3,7 @@ require 'elasticsearch'
|
|
3
3
|
require 'fluent/log-ext'
|
4
4
|
require 'fluent/plugin/input'
|
5
5
|
require_relative 'elasticsearch_constants'
|
6
|
+
require_relative 'elasticsearch_compat'
|
6
7
|
|
7
8
|
module Fluent::Plugin
|
8
9
|
class ElasticsearchInput < Input
|
@@ -218,7 +219,7 @@ module Fluent::Plugin
|
|
218
219
|
|
219
220
|
headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
|
220
221
|
|
221
|
-
transport =
|
222
|
+
transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(
|
222
223
|
connection_options.merge(
|
223
224
|
options: {
|
224
225
|
reload_connections: local_reload_connections,
|
@@ -1,10 +1,11 @@
|
|
1
1
|
require 'oj'
|
2
|
+
require_relative 'elasticsearch_compat'
|
2
3
|
|
3
4
|
module Fluent::Plugin
|
4
5
|
module Serializer
|
5
6
|
|
6
7
|
class Oj
|
7
|
-
include
|
8
|
+
include TRANSPORT_CLASS::Transport::Serializer::Base
|
8
9
|
|
9
10
|
# De-serialize a Hash from JSON string
|
10
11
|
#
|
@@ -3,10 +3,6 @@ require 'date'
|
|
3
3
|
require 'excon'
|
4
4
|
require 'elasticsearch'
|
5
5
|
require 'set'
|
6
|
-
begin
|
7
|
-
require 'elasticsearch/xpack'
|
8
|
-
rescue LoadError
|
9
|
-
end
|
10
6
|
require 'json'
|
11
7
|
require 'uri'
|
12
8
|
require 'base64'
|
@@ -23,6 +19,7 @@ require 'fluent/time'
|
|
23
19
|
require 'fluent/unique_id'
|
24
20
|
require 'fluent/log-ext'
|
25
21
|
require 'zlib'
|
22
|
+
require_relative 'elasticsearch_compat'
|
26
23
|
require_relative 'elasticsearch_constants'
|
27
24
|
require_relative 'elasticsearch_error'
|
28
25
|
require_relative 'elasticsearch_error_handler'
|
@@ -413,11 +410,11 @@ EOC
|
|
413
410
|
end
|
414
411
|
end
|
415
412
|
|
416
|
-
if Gem::Version.create(::
|
413
|
+
if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
417
414
|
if compression
|
418
415
|
raise Fluent::ConfigError, <<-EOC
|
419
416
|
Cannot use compression with elasticsearch-transport plugin version < 7.2.0
|
420
|
-
Your elasticsearch-transport plugin version version is #{
|
417
|
+
Your elasticsearch-transport plugin version version is #{TRANSPORT_CLASS::VERSION}.
|
421
418
|
Please consider to upgrade ES client.
|
422
419
|
EOC
|
423
420
|
end
|
@@ -495,7 +492,11 @@ EOC
|
|
495
492
|
end
|
496
493
|
|
497
494
|
def detect_es_major_version
|
498
|
-
|
495
|
+
begin
|
496
|
+
@_es_info ||= client.info
|
497
|
+
rescue Elasticsearch::UnsupportedProductError => e
|
498
|
+
raise Fluent::ConfigError, "Using Elasticsearch client #{client_library_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
|
499
|
+
end
|
499
500
|
begin
|
500
501
|
unless version = @_es_info.dig("version", "number")
|
501
502
|
version = @default_elasticsearch_version
|
@@ -613,7 +614,7 @@ EOC
|
|
613
614
|
.merge(gzip_headers)
|
614
615
|
ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
|
615
616
|
|
616
|
-
transport =
|
617
|
+
transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
|
617
618
|
options: {
|
618
619
|
reload_connections: local_reload_connections,
|
619
620
|
reload_on_failure: @reload_on_failure,
|
@@ -11,6 +11,9 @@ module Fluent::Plugin
|
|
11
11
|
config_param :data_stream_name, :string
|
12
12
|
config_param :data_stream_ilm_name, :string, :default => nil
|
13
13
|
config_param :data_stream_template_name, :string, :default => nil
|
14
|
+
config_param :data_stream_ilm_policy, :string, :default => nil
|
15
|
+
config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
|
16
|
+
|
14
17
|
# Elasticsearch 7.9 or later always support new style of index template.
|
15
18
|
config_set_default :use_legacy_template, false
|
16
19
|
|
@@ -20,15 +23,24 @@ module Fluent::Plugin
|
|
20
23
|
def configure(conf)
|
21
24
|
super
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
26
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
27
|
+
begin
|
28
|
+
require 'elasticsearch/api'
|
29
|
+
require 'elasticsearch/xpack'
|
30
|
+
rescue LoadError
|
31
|
+
raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
|
32
|
+
end
|
33
|
+
else
|
34
|
+
begin
|
35
|
+
require 'elasticsearch/api'
|
36
|
+
rescue LoadError
|
37
|
+
raise Fluent::ConfigError, "'elasticsearch/api is required for <@elasticsearch_data_stream>."
|
38
|
+
end
|
28
39
|
end
|
29
40
|
|
30
41
|
@data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
|
31
42
|
@data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
|
43
|
+
@data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
|
32
44
|
|
33
45
|
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
34
46
|
unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
|
@@ -38,13 +50,16 @@ module Fluent::Plugin
|
|
38
50
|
@data_stream_names = []
|
39
51
|
end
|
40
52
|
|
41
|
-
@client = client
|
42
53
|
unless @use_placeholder
|
43
54
|
begin
|
44
55
|
@data_stream_names = [@data_stream_name]
|
45
|
-
|
46
|
-
|
47
|
-
|
56
|
+
retry_operate(@max_retry_putting_template,
|
57
|
+
@fail_on_putting_template_retry_exceed,
|
58
|
+
@catch_transport_exception_on_retry) do
|
59
|
+
create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
|
60
|
+
create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
|
61
|
+
create_data_stream(@data_stream_name)
|
62
|
+
end
|
48
63
|
rescue => e
|
49
64
|
raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
|
50
65
|
end
|
@@ -76,21 +91,27 @@ module Fluent::Plugin
|
|
76
91
|
end
|
77
92
|
end
|
78
93
|
|
79
|
-
def create_ilm_policy(datastream_name, template_name, ilm_name, host)
|
80
|
-
|
94
|
+
def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
|
95
|
+
unless @data_stream_ilm_policy_overwrite
|
96
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
|
97
|
+
end
|
98
|
+
|
81
99
|
params = {
|
82
|
-
|
83
|
-
body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
|
100
|
+
body: @data_stream_ilm_policy
|
84
101
|
}
|
85
102
|
retry_operate(@max_retry_putting_template,
|
86
103
|
@fail_on_putting_template_retry_exceed,
|
87
104
|
@catch_transport_exception_on_retry) do
|
88
|
-
|
105
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
106
|
+
client(host).enrich.put_policy(params.merge(name: ilm_name))
|
107
|
+
else
|
108
|
+
client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
|
109
|
+
end
|
89
110
|
end
|
90
111
|
end
|
91
112
|
|
92
|
-
def create_index_template(datastream_name, template_name, ilm_name, host)
|
93
|
-
return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
|
113
|
+
def create_index_template(datastream_name, template_name, ilm_name, host = nil)
|
114
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
94
115
|
body = {
|
95
116
|
"index_patterns" => ["#{datastream_name}*"],
|
96
117
|
"data_stream" => {},
|
@@ -107,38 +128,42 @@ module Fluent::Plugin
|
|
107
128
|
retry_operate(@max_retry_putting_template,
|
108
129
|
@fail_on_putting_template_retry_exceed,
|
109
130
|
@catch_transport_exception_on_retry) do
|
110
|
-
|
131
|
+
client(host).indices.put_index_template(params)
|
111
132
|
end
|
112
133
|
end
|
113
134
|
|
114
|
-
def data_stream_exist?(datastream_name)
|
135
|
+
def data_stream_exist?(datastream_name, host = nil)
|
115
136
|
params = {
|
116
137
|
name: datastream_name
|
117
138
|
}
|
118
139
|
begin
|
119
|
-
response =
|
120
|
-
return (not response.is_a?(
|
121
|
-
rescue
|
140
|
+
response = client(host).indices.get_data_stream(params)
|
141
|
+
return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
|
142
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
|
122
143
|
log.info "Specified data stream does not exist. Will be created: <#{e}>"
|
123
144
|
return false
|
124
145
|
end
|
125
146
|
end
|
126
147
|
|
127
|
-
def create_data_stream(datastream_name)
|
128
|
-
return if data_stream_exist?(datastream_name)
|
148
|
+
def create_data_stream(datastream_name, host = nil)
|
149
|
+
return if data_stream_exist?(datastream_name, host)
|
129
150
|
params = {
|
130
151
|
name: datastream_name
|
131
152
|
}
|
132
153
|
retry_operate(@max_retry_putting_template,
|
133
154
|
@fail_on_putting_template_retry_exceed,
|
134
155
|
@catch_transport_exception_on_retry) do
|
135
|
-
|
156
|
+
client(host).indices.create_data_stream(params)
|
136
157
|
end
|
137
158
|
end
|
138
159
|
|
139
|
-
def ilm_policy_exists?(policy_id)
|
160
|
+
def ilm_policy_exists?(policy_id, host = nil)
|
140
161
|
begin
|
141
|
-
|
162
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
163
|
+
client(host).enrich.get_policy(name: policy_id)
|
164
|
+
else
|
165
|
+
client(host).ilm.get_policy(policy_id: policy_id)
|
166
|
+
end
|
142
167
|
true
|
143
168
|
rescue
|
144
169
|
false
|
@@ -152,7 +177,7 @@ module Fluent::Plugin
|
|
152
177
|
client(host).indices.get_index_template(:name => name)
|
153
178
|
end
|
154
179
|
return true
|
155
|
-
rescue
|
180
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound
|
156
181
|
return false
|
157
182
|
end
|
158
183
|
|
@@ -192,8 +217,9 @@ module Fluent::Plugin
|
|
192
217
|
data_stream_name = @data_stream_name
|
193
218
|
data_stream_template_name = @data_stream_template_name
|
194
219
|
data_stream_ilm_name = @data_stream_ilm_name
|
195
|
-
host =
|
220
|
+
host = nil
|
196
221
|
if @use_placeholder
|
222
|
+
host = extract_placeholders(@host, chunk)
|
197
223
|
data_stream_name = extract_placeholders(@data_stream_name, chunk)
|
198
224
|
data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
|
199
225
|
data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
|
@@ -217,6 +243,10 @@ module Fluent::Plugin
|
|
217
243
|
chunk.msgpack_each do |time, record|
|
218
244
|
next unless record.is_a? Hash
|
219
245
|
|
246
|
+
if @include_tag_key
|
247
|
+
record[@tag_key] = tag
|
248
|
+
end
|
249
|
+
|
220
250
|
begin
|
221
251
|
record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
|
222
252
|
bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
|
@@ -230,7 +260,7 @@ module Fluent::Plugin
|
|
230
260
|
body: bulk_message
|
231
261
|
}
|
232
262
|
begin
|
233
|
-
response =
|
263
|
+
response = client(host).bulk(params)
|
234
264
|
if response['errors']
|
235
265
|
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
|
236
266
|
end
|
@@ -28,6 +28,8 @@ module Fluent::Plugin
|
|
28
28
|
@dynamic_config[key] = value.to_s
|
29
29
|
}
|
30
30
|
# end eval all configs
|
31
|
+
|
32
|
+
log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
|
31
33
|
end
|
32
34
|
|
33
35
|
def create_meta_config_map
|
@@ -53,7 +55,7 @@ module Fluent::Plugin
|
|
53
55
|
end
|
54
56
|
headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
|
55
57
|
ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
|
56
|
-
transport =
|
58
|
+
transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
|
57
59
|
options: {
|
58
60
|
reload_connections: @reload_connections,
|
59
61
|
reload_on_failure: @reload_on_failure,
|
@@ -18,17 +18,25 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
21
|
-
def
|
21
|
+
def elasticsearch_version
|
22
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
23
|
+
TRANSPORT_CLASS::VERSION
|
24
|
+
else
|
25
|
+
'6.4.2'.freeze
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
|
22
30
|
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
23
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
31
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
24
32
|
end
|
25
33
|
|
26
|
-
def stub_elastic_info_not_found(url="http://localhost:9200/", version=
|
27
|
-
stub_request(:get, url).to_return(:status => [404, "Not Found"])
|
34
|
+
def stub_elastic_info_not_found(url="http://localhost:9200/", version=elasticsearch_version)
|
35
|
+
stub_request(:get, url).to_return(:status => [404, "Not Found"], headers: {'x-elastic-product' => 'Elasticsearch' })
|
28
36
|
end
|
29
37
|
|
30
|
-
def stub_elastic_info_unavailable(url="http://localhost:9200/", version=
|
31
|
-
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
|
38
|
+
def stub_elastic_info_unavailable(url="http://localhost:9200/", version=elasticsearch_version)
|
39
|
+
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"], headers: {'x-elastic-product' => 'Elasticsearch' })
|
32
40
|
end
|
33
41
|
|
34
42
|
def sample_record(content={})
|
@@ -60,7 +68,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
|
60
68
|
reload_after 10
|
61
69
|
catch_transport_exception_on_retry false # For fallback testing
|
62
70
|
]
|
63
|
-
assert_raise(
|
71
|
+
assert_raise(TRANSPORT_CLASS::Transport::Errors::NotFound) do
|
64
72
|
driver(config)
|
65
73
|
end
|
66
74
|
driver.run(default_tag: 'test') do
|