fluent-plugin-elasticsearch 5.1.2 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/History.md +15 -0
- data/README.md +16 -1
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +27 -0
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +8 -4
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +2 -1
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +6 -9
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +56 -30
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +1 -1
- data/test/plugin/test_elasticsearch_fallback_selector.rb +15 -7
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +47 -17
- data/test/plugin/test_in_elasticsearch.rb +10 -2
- data/test/plugin/test_out_elasticsearch.rb +220 -122
- data/test/plugin/test_out_elasticsearch_data_stream.rb +187 -33
- data/test/plugin/test_out_elasticsearch_dynamic.rb +57 -20
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4c4d872817c5c5f788b40c89adfddec72dc5765666ab90046a38d5e881ba4f2d
|
4
|
+
data.tar.gz: 8e83854fa3eea144ba97e44ba52885dd5f28ef3801fe293c69cf5158e7839789
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 65eb51a5cd710eb39c54f822f3748bb6ba202ebc87d9aab563adf4befec9696279219f452cc99a1d7528e4c484e03807f195cba8695ec16cc0b3525793a872be
|
7
|
+
data.tar.gz: 3ce726708a2b1900f27999518a56bbf857c28337ce1e3d82ab0acd8fbdb58d92488ddb7e9ccf5a8482768b2c4da1c261b96835f02237321b1a941c6d3b0eb2dc
|
data/Gemfile
CHANGED
@@ -7,5 +7,5 @@ gem 'simplecov', require: false
|
|
7
7
|
gem 'coveralls', ">= 0.8.0", require: false
|
8
8
|
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
9
|
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
|
-
gem "elasticsearch-xpack"
|
10
|
+
gem "elasticsearch-xpack" if ENV["USE_XPACK"]
|
11
11
|
gem "oj"
|
data/History.md
CHANGED
@@ -2,6 +2,21 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 5.2.0
|
6
|
+
- Migrate to handle Elasticsearch 8 (#949)
|
7
|
+
|
8
|
+
### 5.1.5
|
9
|
+
- Make retryable DataStreams creation at configure phase (#943)
|
10
|
+
- Handle @hosts parameter on data_stream plugin (#942)
|
11
|
+
- allow specifying custom ILM policies for data streams (#933)
|
12
|
+
|
13
|
+
### 5.1.4
|
14
|
+
- Handle ES8 or above more strictly (#931)
|
15
|
+
- fixing double "\_policy" in index lifecycle management policy for elasticsearch\_data\_stream output (#930)
|
16
|
+
|
17
|
+
### 5.1.3
|
18
|
+
- fixing execution order for dynamic data stream creation (#928)
|
19
|
+
|
5
20
|
### 5.1.2
|
6
21
|
- Fix default values of datastream parameters (#926)
|
7
22
|
|
data/README.md
CHANGED
@@ -1521,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
|
|
1521
1521
|
data_stream_name test
|
1522
1522
|
```
|
1523
1523
|
|
1524
|
-
When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
|
1524
|
+
When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name` or `data_stream_ilm_policy`, ILM default policy is set to the specified data stream.
|
1525
1525
|
Then, the matching index template is also created automatically.
|
1526
1526
|
|
1527
1527
|
### data_stream_name
|
@@ -1545,6 +1545,21 @@ There are some limitations about naming rule.
|
|
1545
1545
|
|
1546
1546
|
In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).
|
1547
1547
|
|
1548
|
+
|
1549
|
+
### data_stream_ilm_policy
|
1550
|
+
|
1551
|
+
You can specify the ILM policy contents as hash. If not present, it will apply the ILM default policy.
|
1552
|
+
|
1553
|
+
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
1554
|
+
|
1555
|
+
### data_stream_ilm_policy_overwrite
|
1556
|
+
|
1557
|
+
Specify whether the data stream ILM policy should be overwritten.
|
1558
|
+
|
1559
|
+
Default value is `false`.
|
1560
|
+
|
1561
|
+
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
1562
|
+
|
1548
1563
|
## Troubleshooting
|
1549
1564
|
|
1550
1565
|
See [Troubleshooting document](README.Troubleshooting.md)
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '5.
|
6
|
+
s.version = '5.2.0'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -0,0 +1,27 @@
|
|
1
|
+
begin
|
2
|
+
require 'elastic/transport'
|
3
|
+
::TRANSPORT_CLASS = Elastic::Transport
|
4
|
+
rescue LoadError
|
5
|
+
end
|
6
|
+
begin
|
7
|
+
require 'elasticsearch/transport'
|
8
|
+
::TRANSPORT_CLASS = Elasticsearch::Transport
|
9
|
+
rescue LoadError
|
10
|
+
end
|
11
|
+
if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("8.0.0")
|
12
|
+
begin
|
13
|
+
require 'elasticsearch/xpack'
|
14
|
+
rescue LoadError
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
begin
|
19
|
+
require 'elastic/transport/transport/connections/selector'
|
20
|
+
::SELECTOR_CLASS = Elastic::Transport::Transport::Connections::Selector
|
21
|
+
rescue LoadError
|
22
|
+
end
|
23
|
+
begin
|
24
|
+
require 'elasticsearch/transport/transport/connections/selector'
|
25
|
+
::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
|
26
|
+
rescue LoadError
|
27
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
|
-
|
1
|
+
require_relative 'elasticsearch_compat'
|
2
2
|
|
3
3
|
class Fluent::Plugin::ElasticseatchFallbackSelector
|
4
|
-
include
|
4
|
+
include SELECTOR_CLASS::Base
|
5
5
|
|
6
6
|
def select(options={})
|
7
7
|
connections.first
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require_relative 'elasticsearch_compat'
|
2
|
+
|
1
3
|
module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
2
4
|
ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
|
3
5
|
|
@@ -21,7 +23,7 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
21
23
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
|
22
24
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
|
23
25
|
|
24
|
-
rescue
|
26
|
+
rescue ::TRANSPORT_CLASS::Transport::Error => e
|
25
27
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
|
26
28
|
end
|
27
29
|
end
|
@@ -43,12 +45,20 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
43
45
|
end
|
44
46
|
|
45
47
|
def get_ilm_policy
|
46
|
-
|
48
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
49
|
+
client.ilm.get_policy
|
50
|
+
else
|
51
|
+
client.enrich.get_policy
|
52
|
+
end
|
47
53
|
end
|
48
54
|
|
49
55
|
def ilm_policy_exists?(policy_id)
|
50
56
|
begin
|
51
|
-
|
57
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
58
|
+
client.ilm.get_policy(policy_id: policy_id)
|
59
|
+
else
|
60
|
+
client.enrich.get_policy(name: policy_id)
|
61
|
+
end
|
52
62
|
true
|
53
63
|
rescue
|
54
64
|
false
|
@@ -57,7 +67,11 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
57
67
|
|
58
68
|
def ilm_policy_put(policy_id, policy)
|
59
69
|
log.info("Installing ILM policy: #{policy}")
|
60
|
-
|
70
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
71
|
+
client.ilm.put_policy(policy_id: policy_id, body: policy)
|
72
|
+
else
|
73
|
+
client.enrich.put_policy(name: policy_id, body: policy)
|
74
|
+
end
|
61
75
|
end
|
62
76
|
|
63
77
|
def default_policy_payload
|
@@ -28,12 +28,16 @@ module Fluent::ElasticsearchIndexTemplate
|
|
28
28
|
client(host).indices.get_index_template(:name => name)
|
29
29
|
end
|
30
30
|
return true
|
31
|
-
rescue
|
31
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound
|
32
32
|
return false
|
33
33
|
end
|
34
34
|
|
35
35
|
def host_unreachable_exceptions
|
36
|
-
if Gem::Version.new(::
|
36
|
+
if Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
37
|
+
# elasticsearch-ruby 8.0.0's elastic-transport uses
|
38
|
+
# direct callable #host_unreachable_exceptions again.
|
39
|
+
client.transport.host_unreachable_exceptions
|
40
|
+
elsif Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
37
41
|
# elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
|
38
42
|
# Elasticsearch class on Transport.
|
39
43
|
# This is why #host_unreachable_exceptions is not callable directly
|
@@ -47,7 +51,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
47
51
|
def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
|
48
52
|
return unless block_given?
|
49
53
|
retries = 0
|
50
|
-
transport_errors =
|
54
|
+
transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map{ |c| TRANSPORT_CLASS::Transport::Errors.const_get c } if catch_trasport_exceptions
|
51
55
|
begin
|
52
56
|
yield
|
53
57
|
rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
|
@@ -78,7 +82,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
78
82
|
|
79
83
|
def indexcreation(index_name, host = nil)
|
80
84
|
client(host).indices.create(:index => index_name)
|
81
|
-
rescue
|
85
|
+
rescue TRANSPORT_CLASS::Transport::Error => e
|
82
86
|
if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
|
83
87
|
log.debug("Index #{index_name} already exists")
|
84
88
|
else
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'elasticsearch'
|
2
|
+
require_relative 'elasticsearch_compat'
|
2
3
|
|
3
|
-
class Fluent::Plugin::ElasticsearchSimpleSniffer <
|
4
|
+
class Fluent::Plugin::ElasticsearchSimpleSniffer < TRANSPORT_CLASS::Transport::Sniffer
|
4
5
|
|
5
6
|
def hosts
|
6
7
|
@transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
|
@@ -3,6 +3,7 @@ require 'elasticsearch'
|
|
3
3
|
require 'fluent/log-ext'
|
4
4
|
require 'fluent/plugin/input'
|
5
5
|
require_relative 'elasticsearch_constants'
|
6
|
+
require_relative 'elasticsearch_compat'
|
6
7
|
|
7
8
|
module Fluent::Plugin
|
8
9
|
class ElasticsearchInput < Input
|
@@ -218,7 +219,7 @@ module Fluent::Plugin
|
|
218
219
|
|
219
220
|
headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
|
220
221
|
|
221
|
-
transport =
|
222
|
+
transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(
|
222
223
|
connection_options.merge(
|
223
224
|
options: {
|
224
225
|
reload_connections: local_reload_connections,
|
@@ -1,10 +1,11 @@
|
|
1
1
|
require 'oj'
|
2
|
+
require_relative 'elasticsearch_compat'
|
2
3
|
|
3
4
|
module Fluent::Plugin
|
4
5
|
module Serializer
|
5
6
|
|
6
7
|
class Oj
|
7
|
-
include
|
8
|
+
include TRANSPORT_CLASS::Transport::Serializer::Base
|
8
9
|
|
9
10
|
# De-serialize a Hash from JSON string
|
10
11
|
#
|
@@ -3,10 +3,6 @@ require 'date'
|
|
3
3
|
require 'excon'
|
4
4
|
require 'elasticsearch'
|
5
5
|
require 'set'
|
6
|
-
begin
|
7
|
-
require 'elasticsearch/xpack'
|
8
|
-
rescue LoadError
|
9
|
-
end
|
10
6
|
require 'json'
|
11
7
|
require 'uri'
|
12
8
|
require 'base64'
|
@@ -23,6 +19,7 @@ require 'fluent/time'
|
|
23
19
|
require 'fluent/unique_id'
|
24
20
|
require 'fluent/log-ext'
|
25
21
|
require 'zlib'
|
22
|
+
require_relative 'elasticsearch_compat'
|
26
23
|
require_relative 'elasticsearch_constants'
|
27
24
|
require_relative 'elasticsearch_error'
|
28
25
|
require_relative 'elasticsearch_error_handler'
|
@@ -413,11 +410,11 @@ EOC
|
|
413
410
|
end
|
414
411
|
end
|
415
412
|
|
416
|
-
if Gem::Version.create(::
|
413
|
+
if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
417
414
|
if compression
|
418
415
|
raise Fluent::ConfigError, <<-EOC
|
419
416
|
Cannot use compression with elasticsearch-transport plugin version < 7.2.0
|
420
|
-
Your elasticsearch-transport plugin version version is #{
|
417
|
+
Your elasticsearch-transport plugin version version is #{TRANSPORT_CLASS::VERSION}.
|
421
418
|
Please consider to upgrade ES client.
|
422
419
|
EOC
|
423
420
|
end
|
@@ -613,7 +610,7 @@ EOC
|
|
613
610
|
.merge(gzip_headers)
|
614
611
|
ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
|
615
612
|
|
616
|
-
transport =
|
613
|
+
transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
|
617
614
|
options: {
|
618
615
|
reload_connections: local_reload_connections,
|
619
616
|
reload_on_failure: @reload_on_failure,
|
@@ -987,12 +984,12 @@ EOC
|
|
987
984
|
elsif @last_seen_major_version == 7
|
988
985
|
log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
|
989
986
|
target_type = '_doc'.freeze
|
990
|
-
elsif @last_seen_major_version >=8
|
987
|
+
elsif @last_seen_major_version >= 8
|
991
988
|
log.debug "Detected ES 8.x or above: document type will not be used."
|
992
989
|
target_type = nil
|
993
990
|
end
|
994
991
|
else
|
995
|
-
if @suppress_type_name && @last_seen_major_version
|
992
|
+
if @suppress_type_name && @last_seen_major_version == 7
|
996
993
|
target_type = nil
|
997
994
|
elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
998
995
|
log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
|
@@ -11,6 +11,9 @@ module Fluent::Plugin
|
|
11
11
|
config_param :data_stream_name, :string
|
12
12
|
config_param :data_stream_ilm_name, :string, :default => nil
|
13
13
|
config_param :data_stream_template_name, :string, :default => nil
|
14
|
+
config_param :data_stream_ilm_policy, :string, :default => nil
|
15
|
+
config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
|
16
|
+
|
14
17
|
# Elasticsearch 7.9 or later always support new style of index template.
|
15
18
|
config_set_default :use_legacy_template, false
|
16
19
|
|
@@ -20,15 +23,24 @@ module Fluent::Plugin
|
|
20
23
|
def configure(conf)
|
21
24
|
super
|
22
25
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
26
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
27
|
+
begin
|
28
|
+
require 'elasticsearch/api'
|
29
|
+
require 'elasticsearch/xpack'
|
30
|
+
rescue LoadError
|
31
|
+
raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
|
32
|
+
end
|
33
|
+
else
|
34
|
+
begin
|
35
|
+
require 'elasticsearch/api'
|
36
|
+
rescue LoadError
|
37
|
+
raise Fluent::ConfigError, "'elasticsearch/api is required for <@elasticsearch_data_stream>."
|
38
|
+
end
|
28
39
|
end
|
29
40
|
|
30
41
|
@data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
|
31
42
|
@data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
|
43
|
+
@data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
|
32
44
|
|
33
45
|
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
34
46
|
unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
|
@@ -38,13 +50,16 @@ module Fluent::Plugin
|
|
38
50
|
@data_stream_names = []
|
39
51
|
end
|
40
52
|
|
41
|
-
@client = client
|
42
53
|
unless @use_placeholder
|
43
54
|
begin
|
44
55
|
@data_stream_names = [@data_stream_name]
|
45
|
-
|
46
|
-
|
47
|
-
|
56
|
+
retry_operate(@max_retry_putting_template,
|
57
|
+
@fail_on_putting_template_retry_exceed,
|
58
|
+
@catch_transport_exception_on_retry) do
|
59
|
+
create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
|
60
|
+
create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
|
61
|
+
create_data_stream(@data_stream_name)
|
62
|
+
end
|
48
63
|
rescue => e
|
49
64
|
raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
|
50
65
|
end
|
@@ -76,21 +91,27 @@ module Fluent::Plugin
|
|
76
91
|
end
|
77
92
|
end
|
78
93
|
|
79
|
-
def create_ilm_policy(datastream_name, template_name, ilm_name, host)
|
80
|
-
|
94
|
+
def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
|
95
|
+
unless @data_stream_ilm_policy_overwrite
|
96
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
|
97
|
+
end
|
98
|
+
|
81
99
|
params = {
|
82
|
-
|
83
|
-
body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
|
100
|
+
body: @data_stream_ilm_policy
|
84
101
|
}
|
85
102
|
retry_operate(@max_retry_putting_template,
|
86
103
|
@fail_on_putting_template_retry_exceed,
|
87
104
|
@catch_transport_exception_on_retry) do
|
88
|
-
|
105
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
106
|
+
client(host).enrich.put_policy(params.merge(name: ilm_name))
|
107
|
+
else
|
108
|
+
client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
|
109
|
+
end
|
89
110
|
end
|
90
111
|
end
|
91
112
|
|
92
|
-
def create_index_template(datastream_name, template_name, ilm_name, host)
|
93
|
-
return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
|
113
|
+
def create_index_template(datastream_name, template_name, ilm_name, host = nil)
|
114
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
94
115
|
body = {
|
95
116
|
"index_patterns" => ["#{datastream_name}*"],
|
96
117
|
"data_stream" => {},
|
@@ -107,38 +128,42 @@ module Fluent::Plugin
|
|
107
128
|
retry_operate(@max_retry_putting_template,
|
108
129
|
@fail_on_putting_template_retry_exceed,
|
109
130
|
@catch_transport_exception_on_retry) do
|
110
|
-
|
131
|
+
client(host).indices.put_index_template(params)
|
111
132
|
end
|
112
133
|
end
|
113
134
|
|
114
|
-
def data_stream_exist?(datastream_name)
|
135
|
+
def data_stream_exist?(datastream_name, host = nil)
|
115
136
|
params = {
|
116
137
|
name: datastream_name
|
117
138
|
}
|
118
139
|
begin
|
119
|
-
response =
|
120
|
-
return (not response.is_a?(
|
121
|
-
rescue
|
140
|
+
response = client(host).indices.get_data_stream(params)
|
141
|
+
return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
|
142
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
|
122
143
|
log.info "Specified data stream does not exist. Will be created: <#{e}>"
|
123
144
|
return false
|
124
145
|
end
|
125
146
|
end
|
126
147
|
|
127
|
-
def create_data_stream(datastream_name)
|
128
|
-
return if data_stream_exist?(datastream_name)
|
148
|
+
def create_data_stream(datastream_name, host = nil)
|
149
|
+
return if data_stream_exist?(datastream_name, host)
|
129
150
|
params = {
|
130
151
|
name: datastream_name
|
131
152
|
}
|
132
153
|
retry_operate(@max_retry_putting_template,
|
133
154
|
@fail_on_putting_template_retry_exceed,
|
134
155
|
@catch_transport_exception_on_retry) do
|
135
|
-
|
156
|
+
client(host).indices.create_data_stream(params)
|
136
157
|
end
|
137
158
|
end
|
138
159
|
|
139
|
-
def ilm_policy_exists?(policy_id)
|
160
|
+
def ilm_policy_exists?(policy_id, host = nil)
|
140
161
|
begin
|
141
|
-
|
162
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
163
|
+
client(host).enrich.get_policy(name: policy_id)
|
164
|
+
else
|
165
|
+
client(host).ilm.get_policy(policy_id: policy_id)
|
166
|
+
end
|
142
167
|
true
|
143
168
|
rescue
|
144
169
|
false
|
@@ -152,7 +177,7 @@ module Fluent::Plugin
|
|
152
177
|
client(host).indices.get_index_template(:name => name)
|
153
178
|
end
|
154
179
|
return true
|
155
|
-
rescue
|
180
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound
|
156
181
|
return false
|
157
182
|
end
|
158
183
|
|
@@ -192,16 +217,17 @@ module Fluent::Plugin
|
|
192
217
|
data_stream_name = @data_stream_name
|
193
218
|
data_stream_template_name = @data_stream_template_name
|
194
219
|
data_stream_ilm_name = @data_stream_ilm_name
|
195
|
-
host =
|
220
|
+
host = nil
|
196
221
|
if @use_placeholder
|
222
|
+
host = extract_placeholders(@host, chunk)
|
197
223
|
data_stream_name = extract_placeholders(@data_stream_name, chunk)
|
198
224
|
data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
|
199
225
|
data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
|
200
226
|
unless @data_stream_names.include?(data_stream_name)
|
201
227
|
begin
|
202
|
-
create_data_stream(data_stream_name)
|
203
228
|
create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
|
204
229
|
create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
|
230
|
+
create_data_stream(data_stream_name)
|
205
231
|
@data_stream_names << data_stream_name
|
206
232
|
rescue => e
|
207
233
|
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
@@ -230,7 +256,7 @@ module Fluent::Plugin
|
|
230
256
|
body: bulk_message
|
231
257
|
}
|
232
258
|
begin
|
233
|
-
response =
|
259
|
+
response = client(host).bulk(params)
|
234
260
|
if response['errors']
|
235
261
|
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
|
236
262
|
end
|
@@ -53,7 +53,7 @@ module Fluent::Plugin
|
|
53
53
|
end
|
54
54
|
headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
|
55
55
|
ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
|
56
|
-
transport =
|
56
|
+
transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
|
57
57
|
options: {
|
58
58
|
reload_connections: @reload_connections,
|
59
59
|
reload_on_failure: @reload_on_failure,
|
@@ -18,17 +18,25 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
21
|
-
def
|
21
|
+
def elasticsearch_version
|
22
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
23
|
+
TRANSPORT_CLASS::VERSION
|
24
|
+
else
|
25
|
+
'6.4.2'.freeze
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
|
22
30
|
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
23
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
31
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
24
32
|
end
|
25
33
|
|
26
|
-
def stub_elastic_info_not_found(url="http://localhost:9200/", version=
|
27
|
-
stub_request(:get, url).to_return(:status => [404, "Not Found"])
|
34
|
+
def stub_elastic_info_not_found(url="http://localhost:9200/", version=elasticsearch_version)
|
35
|
+
stub_request(:get, url).to_return(:status => [404, "Not Found"], headers: {'x-elastic-product' => 'Elasticsearch' })
|
28
36
|
end
|
29
37
|
|
30
|
-
def stub_elastic_info_unavailable(url="http://localhost:9200/", version=
|
31
|
-
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
|
38
|
+
def stub_elastic_info_unavailable(url="http://localhost:9200/", version=elasticsearch_version)
|
39
|
+
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"], headers: {'x-elastic-product' => 'Elasticsearch' })
|
32
40
|
end
|
33
41
|
|
34
42
|
def sample_record(content={})
|
@@ -60,7 +68,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
|
60
68
|
reload_after 10
|
61
69
|
catch_transport_exception_on_retry false # For fallback testing
|
62
70
|
]
|
63
|
-
assert_raise(
|
71
|
+
assert_raise(TRANSPORT_CLASS::Transport::Errors::NotFound) do
|
64
72
|
driver(config)
|
65
73
|
end
|
66
74
|
driver.run(default_tag: 'test') do
|