fluent-plugin-elasticsearch 5.1.4 → 5.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9cbe785cfd5534abf0b55d8207eae176d1dd02d5ef7a3092e9f0ef3c6d588e7f
4
- data.tar.gz: c12b413d1415ac38a6d119ff1f0ca106b1d4f1017f206ce72d97a74abe819551
3
+ metadata.gz: 425652fd5cb31671a87378fdaaf8fa911aa6bda0743fe44c53d7631d5018aebb
4
+ data.tar.gz: d3fb4697a32f8448fdd9995adeef50a96839eb15b10f984a03618044c7a9ff35
5
5
  SHA512:
6
- metadata.gz: d30472143db2451da2eca168b22086d157abb7e62c26f1df5afa2ab4ed23a250a2b81fd04424f260dbfe996cc0a4794504e0c4262efc7969db363be02c79bcf9
7
- data.tar.gz: 9fd7a57fe732f18b6f568a96052ba7a5bf51423e93072ba230c7a57866b828dfe07e16b11f937c5d84d0cb01619fe027ccd80c0a0a4fb939d94a41f127ff5e3c
6
+ metadata.gz: d8b6b4e919d79de0153707bfb84033ea403c0a88f7d76a7f1d986a45e67a0fec340762bd0a5c8776e2f6dff18b267550623fe516018a634fa3c56b9f63a3937c
7
+ data.tar.gz: 40b798377a202b4e7c0c0e6f7e84910cf7574711a83a0919c274ff5b8b9eb0dd5b90bd3e5c682ba2be9d684384c29bbbfe67d62d9307e8d0df050ee92d460c0b
data/Gemfile CHANGED
@@ -7,5 +7,5 @@ gem 'simplecov', require: false
7
7
  gem 'coveralls', ">= 0.8.0", require: false
8
8
  gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
9
9
  gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
10
- gem "elasticsearch-xpack"
10
+ gem "elasticsearch-xpack" if ENV["USE_XPACK"]
11
11
  gem "oj"
data/History.md CHANGED
@@ -2,6 +2,19 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 5.2.1
6
+ - respect include\_tag\_key and tag\_key setting when using data streams (#936)
7
+ - Handle unsupported version error (#956)
8
+ - Display deprecated warning on ES dynamic plugin (#955)
9
+
10
+ ### 5.2.0
11
+ - Migrate to handle Elasticsearch 8 (#949)
12
+
13
+ ### 5.1.5
14
+ - Make retryable DataStreams creation at configure phase (#943)
15
+ - Handle @hosts parameter on data_stream plugin (#942)
16
+ - allow specifying custom ILM policies for data streams (#933)
17
+
5
18
  ### 5.1.4
6
19
  - Handle ES8 or above more strictly (#931)
7
20
  - fixing double "\_policy" in index lifecycle management policy for elasticsearch\_data\_stream output (#930)
data/README.md CHANGED
@@ -1521,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
1521
1521
  data_stream_name test
1522
1522
  ```
1523
1523
 
1524
- When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
1524
+ When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name` or `data_stream_ilm_policy`, ILM default policy is set to the specified data stream.
1525
1525
  Then, the matching index template is also created automatically.
1526
1526
 
1527
1527
  ### data_stream_name
@@ -1545,6 +1545,21 @@ There are some limitations about naming rule.
1545
1545
 
1546
1546
  In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).
1547
1547
 
1548
+
1549
+ ### data_stream_ilm_policy
1550
+
1551
+ You can specify the ILM policy contents as hash. If not present, it will apply the ILM default policy.
1552
+
1553
+ **NOTE:** This parameter requests to install elasticsearch-xpack gem.
1554
+
1555
+ ### data_stream_ilm_policy_overwrite
1556
+
1557
+ Specify whether the data stream ILM policy should be overwritten.
1558
+
1559
+ Default value is `false`.
1560
+
1561
+ **NOTE:** This parameter requests to install elasticsearch-xpack gem.
1562
+
1548
1563
  ## Troubleshooting
1549
1564
 
1550
1565
  See [Troubleshooting document](README.Troubleshooting.md)
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '5.1.4'
6
+ s.version = '5.2.1'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -0,0 +1,30 @@
1
+ begin
2
+ require 'elastic/transport'
3
+ ::TRANSPORT_CLASS = Elastic::Transport
4
+ rescue LoadError
5
+ end
6
+ begin
7
+ require 'elasticsearch/transport'
8
+ ::TRANSPORT_CLASS = Elasticsearch::Transport
9
+ rescue LoadError
10
+ end
11
+ if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("8.0.0")
12
+ begin
13
+ require 'elasticsearch/xpack'
14
+ rescue LoadError
15
+ end
16
+ end
17
+
18
+ begin
19
+ require 'elastic/transport/transport/connections/selector'
20
+ ::SELECTOR_CLASS = Elastic::Transport::Transport::Connections::Selector
21
+ rescue LoadError
22
+ end
23
+ begin
24
+ require 'elasticsearch/transport/transport/connections/selector'
25
+ ::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
26
+ rescue LoadError
27
+ end
28
+ unless defined?(Elasticsearch::UnsupportedProductError)
29
+ class Elasticsearch::UnsupportedProductError < StandardError; end
30
+ end
@@ -1,7 +1,7 @@
1
- require 'elasticsearch/transport/transport/connections/selector'
1
+ require_relative 'elasticsearch_compat'
2
2
 
3
3
  class Fluent::Plugin::ElasticseatchFallbackSelector
4
- include Elasticsearch::Transport::Transport::Connections::Selector::Base
4
+ include SELECTOR_CLASS::Base
5
5
 
6
6
  def select(options={})
7
7
  connections.first
@@ -1,3 +1,5 @@
1
+ require_relative 'elasticsearch_compat'
2
+
1
3
  module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
2
4
  ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
3
5
 
@@ -21,7 +23,7 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
21
23
  raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
22
24
  raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
23
25
 
24
- rescue Elasticsearch::Transport::Transport::Error => e
26
+ rescue ::TRANSPORT_CLASS::Transport::Error => e
25
27
  raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
26
28
  end
27
29
  end
@@ -43,12 +45,20 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
43
45
  end
44
46
 
45
47
  def get_ilm_policy
46
- client.ilm.get_policy
48
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
49
+ client.ilm.get_policy
50
+ else
51
+ client.enrich.get_policy
52
+ end
47
53
  end
48
54
 
49
55
  def ilm_policy_exists?(policy_id)
50
56
  begin
51
- client.ilm.get_policy(policy_id: policy_id)
57
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
58
+ client.ilm.get_policy(policy_id: policy_id)
59
+ else
60
+ client.enrich.get_policy(name: policy_id)
61
+ end
52
62
  true
53
63
  rescue
54
64
  false
@@ -57,7 +67,11 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
57
67
 
58
68
  def ilm_policy_put(policy_id, policy)
59
69
  log.info("Installing ILM policy: #{policy}")
60
- client.ilm.put_policy(policy_id: policy_id, body: policy)
70
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
71
+ client.ilm.put_policy(policy_id: policy_id, body: policy)
72
+ else
73
+ client.enrich.put_policy(name: policy_id, body: policy)
74
+ end
61
75
  end
62
76
 
63
77
  def default_policy_payload
@@ -28,12 +28,16 @@ module Fluent::ElasticsearchIndexTemplate
28
28
  client(host).indices.get_index_template(:name => name)
29
29
  end
30
30
  return true
31
- rescue Elasticsearch::Transport::Transport::Errors::NotFound
31
+ rescue TRANSPORT_CLASS::Transport::Errors::NotFound
32
32
  return false
33
33
  end
34
34
 
35
35
  def host_unreachable_exceptions
36
- if Gem::Version.new(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
36
+ if Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
37
+ # elasticsearch-ruby 8.0.0's elastic-transport uses
38
+ # direct callable #host_unreachable_exceptions again.
39
+ client.transport.host_unreachable_exceptions
40
+ elsif Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
37
41
  # elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
38
42
  # Elasticsearch class on Transport.
39
43
  # This is why #host_unreachable_exceptions is not callable directly
@@ -47,7 +51,7 @@ module Fluent::ElasticsearchIndexTemplate
47
51
  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
48
52
  return unless block_given?
49
53
  retries = 0
50
- transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
54
+ transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map{ |c| TRANSPORT_CLASS::Transport::Errors.const_get c } if catch_trasport_exceptions
51
55
  begin
52
56
  yield
53
57
  rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
@@ -78,7 +82,7 @@ module Fluent::ElasticsearchIndexTemplate
78
82
 
79
83
  def indexcreation(index_name, host = nil)
80
84
  client(host).indices.create(:index => index_name)
81
- rescue Elasticsearch::Transport::Transport::Error => e
85
+ rescue TRANSPORT_CLASS::Transport::Error => e
82
86
  if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
83
87
  log.debug("Index #{index_name} already exists")
84
88
  else
@@ -1,6 +1,7 @@
1
1
  require 'elasticsearch'
2
+ require_relative 'elasticsearch_compat'
2
3
 
3
- class Fluent::Plugin::ElasticsearchSimpleSniffer < Elasticsearch::Transport::Transport::Sniffer
4
+ class Fluent::Plugin::ElasticsearchSimpleSniffer < TRANSPORT_CLASS::Transport::Sniffer
4
5
 
5
6
  def hosts
6
7
  @transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
@@ -3,6 +3,7 @@ require 'elasticsearch'
3
3
  require 'fluent/log-ext'
4
4
  require 'fluent/plugin/input'
5
5
  require_relative 'elasticsearch_constants'
6
+ require_relative 'elasticsearch_compat'
6
7
 
7
8
  module Fluent::Plugin
8
9
  class ElasticsearchInput < Input
@@ -218,7 +219,7 @@ module Fluent::Plugin
218
219
 
219
220
  headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
220
221
 
221
- transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(
222
+ transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(
222
223
  connection_options.merge(
223
224
  options: {
224
225
  reload_connections: local_reload_connections,
@@ -1,10 +1,11 @@
1
1
  require 'oj'
2
+ require_relative 'elasticsearch_compat'
2
3
 
3
4
  module Fluent::Plugin
4
5
  module Serializer
5
6
 
6
7
  class Oj
7
- include Elasticsearch::Transport::Transport::Serializer::Base
8
+ include TRANSPORT_CLASS::Transport::Serializer::Base
8
9
 
9
10
  # De-serialize a Hash from JSON string
10
11
  #
@@ -3,10 +3,6 @@ require 'date'
3
3
  require 'excon'
4
4
  require 'elasticsearch'
5
5
  require 'set'
6
- begin
7
- require 'elasticsearch/xpack'
8
- rescue LoadError
9
- end
10
6
  require 'json'
11
7
  require 'uri'
12
8
  require 'base64'
@@ -23,6 +19,7 @@ require 'fluent/time'
23
19
  require 'fluent/unique_id'
24
20
  require 'fluent/log-ext'
25
21
  require 'zlib'
22
+ require_relative 'elasticsearch_compat'
26
23
  require_relative 'elasticsearch_constants'
27
24
  require_relative 'elasticsearch_error'
28
25
  require_relative 'elasticsearch_error_handler'
@@ -413,11 +410,11 @@ EOC
413
410
  end
414
411
  end
415
412
 
416
- if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
413
+ if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
417
414
  if compression
418
415
  raise Fluent::ConfigError, <<-EOC
419
416
  Cannot use compression with elasticsearch-transport plugin version < 7.2.0
420
- Your elasticsearch-transport plugin version version is #{Elasticsearch::Transport::VERSION}.
417
+ Your elasticsearch-transport plugin version version is #{TRANSPORT_CLASS::VERSION}.
421
418
  Please consider to upgrade ES client.
422
419
  EOC
423
420
  end
@@ -495,7 +492,11 @@ EOC
495
492
  end
496
493
 
497
494
  def detect_es_major_version
498
- @_es_info ||= client.info
495
+ begin
496
+ @_es_info ||= client.info
497
+ rescue Elasticsearch::UnsupportedProductError => e
498
+ raise Fluent::ConfigError, "Using Elasticsearch client #{client_library_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
499
+ end
499
500
  begin
500
501
  unless version = @_es_info.dig("version", "number")
501
502
  version = @default_elasticsearch_version
@@ -613,7 +614,7 @@ EOC
613
614
  .merge(gzip_headers)
614
615
  ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
615
616
 
616
- transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
617
+ transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
617
618
  options: {
618
619
  reload_connections: local_reload_connections,
619
620
  reload_on_failure: @reload_on_failure,
@@ -11,6 +11,9 @@ module Fluent::Plugin
11
11
  config_param :data_stream_name, :string
12
12
  config_param :data_stream_ilm_name, :string, :default => nil
13
13
  config_param :data_stream_template_name, :string, :default => nil
14
+ config_param :data_stream_ilm_policy, :string, :default => nil
15
+ config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
16
+
14
17
  # Elasticsearch 7.9 or later always support new style of index template.
15
18
  config_set_default :use_legacy_template, false
16
19
 
@@ -20,15 +23,24 @@ module Fluent::Plugin
20
23
  def configure(conf)
21
24
  super
22
25
 
23
- begin
24
- require 'elasticsearch/api'
25
- require 'elasticsearch/xpack'
26
- rescue LoadError
27
- raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
26
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
27
+ begin
28
+ require 'elasticsearch/api'
29
+ require 'elasticsearch/xpack'
30
+ rescue LoadError
31
+ raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
32
+ end
33
+ else
34
+ begin
35
+ require 'elasticsearch/api'
36
+ rescue LoadError
37
+ raise Fluent::ConfigError, "'elasticsearch/api is required for <@elasticsearch_data_stream>."
38
+ end
28
39
  end
29
40
 
30
41
  @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
31
42
  @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
43
+ @data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
32
44
 
33
45
  # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
34
46
  unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
@@ -38,13 +50,16 @@ module Fluent::Plugin
38
50
  @data_stream_names = []
39
51
  end
40
52
 
41
- @client = client
42
53
  unless @use_placeholder
43
54
  begin
44
55
  @data_stream_names = [@data_stream_name]
45
- create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
46
- create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
47
- create_data_stream(@data_stream_name)
56
+ retry_operate(@max_retry_putting_template,
57
+ @fail_on_putting_template_retry_exceed,
58
+ @catch_transport_exception_on_retry) do
59
+ create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
60
+ create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
61
+ create_data_stream(@data_stream_name)
62
+ end
48
63
  rescue => e
49
64
  raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
50
65
  end
@@ -76,21 +91,27 @@ module Fluent::Plugin
76
91
  end
77
92
  end
78
93
 
79
- def create_ilm_policy(datastream_name, template_name, ilm_name, host)
80
- return if data_stream_exist?(datastream_name) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name)
94
+ def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
95
+ unless @data_stream_ilm_policy_overwrite
96
+ return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
97
+ end
98
+
81
99
  params = {
82
- policy_id: ilm_name,
83
- body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
100
+ body: @data_stream_ilm_policy
84
101
  }
85
102
  retry_operate(@max_retry_putting_template,
86
103
  @fail_on_putting_template_retry_exceed,
87
104
  @catch_transport_exception_on_retry) do
88
- @client.xpack.ilm.put_policy(params)
105
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
106
+ client(host).enrich.put_policy(params.merge(name: ilm_name))
107
+ else
108
+ client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
109
+ end
89
110
  end
90
111
  end
91
112
 
92
- def create_index_template(datastream_name, template_name, ilm_name, host)
93
- return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
113
+ def create_index_template(datastream_name, template_name, ilm_name, host = nil)
114
+ return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
94
115
  body = {
95
116
  "index_patterns" => ["#{datastream_name}*"],
96
117
  "data_stream" => {},
@@ -107,38 +128,42 @@ module Fluent::Plugin
107
128
  retry_operate(@max_retry_putting_template,
108
129
  @fail_on_putting_template_retry_exceed,
109
130
  @catch_transport_exception_on_retry) do
110
- @client.indices.put_index_template(params)
131
+ client(host).indices.put_index_template(params)
111
132
  end
112
133
  end
113
134
 
114
- def data_stream_exist?(datastream_name)
135
+ def data_stream_exist?(datastream_name, host = nil)
115
136
  params = {
116
137
  name: datastream_name
117
138
  }
118
139
  begin
119
- response = @client.indices.get_data_stream(params)
120
- return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
121
- rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
140
+ response = client(host).indices.get_data_stream(params)
141
+ return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
142
+ rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
122
143
  log.info "Specified data stream does not exist. Will be created: <#{e}>"
123
144
  return false
124
145
  end
125
146
  end
126
147
 
127
- def create_data_stream(datastream_name)
128
- return if data_stream_exist?(datastream_name)
148
+ def create_data_stream(datastream_name, host = nil)
149
+ return if data_stream_exist?(datastream_name, host)
129
150
  params = {
130
151
  name: datastream_name
131
152
  }
132
153
  retry_operate(@max_retry_putting_template,
133
154
  @fail_on_putting_template_retry_exceed,
134
155
  @catch_transport_exception_on_retry) do
135
- @client.indices.create_data_stream(params)
156
+ client(host).indices.create_data_stream(params)
136
157
  end
137
158
  end
138
159
 
139
- def ilm_policy_exists?(policy_id)
160
+ def ilm_policy_exists?(policy_id, host = nil)
140
161
  begin
141
- @client.ilm.get_policy(policy_id: policy_id)
162
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
163
+ client(host).enrich.get_policy(name: policy_id)
164
+ else
165
+ client(host).ilm.get_policy(policy_id: policy_id)
166
+ end
142
167
  true
143
168
  rescue
144
169
  false
@@ -152,7 +177,7 @@ module Fluent::Plugin
152
177
  client(host).indices.get_index_template(:name => name)
153
178
  end
154
179
  return true
155
- rescue Elasticsearch::Transport::Transport::Errors::NotFound
180
+ rescue TRANSPORT_CLASS::Transport::Errors::NotFound
156
181
  return false
157
182
  end
158
183
 
@@ -192,8 +217,9 @@ module Fluent::Plugin
192
217
  data_stream_name = @data_stream_name
193
218
  data_stream_template_name = @data_stream_template_name
194
219
  data_stream_ilm_name = @data_stream_ilm_name
195
- host = @host
220
+ host = nil
196
221
  if @use_placeholder
222
+ host = extract_placeholders(@host, chunk)
197
223
  data_stream_name = extract_placeholders(@data_stream_name, chunk)
198
224
  data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
199
225
  data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
@@ -217,6 +243,10 @@ module Fluent::Plugin
217
243
  chunk.msgpack_each do |time, record|
218
244
  next unless record.is_a? Hash
219
245
 
246
+ if @include_tag_key
247
+ record[@tag_key] = tag
248
+ end
249
+
220
250
  begin
221
251
  record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
222
252
  bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
@@ -230,7 +260,7 @@ module Fluent::Plugin
230
260
  body: bulk_message
231
261
  }
232
262
  begin
233
- response = @client.bulk(params)
263
+ response = client(host).bulk(params)
234
264
  if response['errors']
235
265
  log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
236
266
  end
@@ -28,6 +28,8 @@ module Fluent::Plugin
28
28
  @dynamic_config[key] = value.to_s
29
29
  }
30
30
  # end eval all configs
31
+
32
+ log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
31
33
  end
32
34
 
33
35
  def create_meta_config_map
@@ -53,7 +55,7 @@ module Fluent::Plugin
53
55
  end
54
56
  headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
55
57
  ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
56
- transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
58
+ transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
57
59
  options: {
58
60
  reload_connections: @reload_connections,
59
61
  reload_on_failure: @reload_on_failure,
@@ -18,17 +18,25 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
18
18
  end
19
19
  end
20
20
 
21
- def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
21
+ def elasticsearch_version
22
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
23
+ TRANSPORT_CLASS::VERSION
24
+ else
25
+ '6.4.2'.freeze
26
+ end
27
+ end
28
+
29
+ def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
22
30
  body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
23
- stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
31
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
24
32
  end
25
33
 
26
- def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
27
- stub_request(:get, url).to_return(:status => [404, "Not Found"])
34
+ def stub_elastic_info_not_found(url="http://localhost:9200/", version=elasticsearch_version)
35
+ stub_request(:get, url).to_return(:status => [404, "Not Found"], headers: {'x-elastic-product' => 'Elasticsearch' })
28
36
  end
29
37
 
30
- def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
31
- stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
38
+ def stub_elastic_info_unavailable(url="http://localhost:9200/", version=elasticsearch_version)
39
+ stub_request(:get, url).to_return(:status => [503, "Service Unavailable"], headers: {'x-elastic-product' => 'Elasticsearch' })
32
40
  end
33
41
 
34
42
  def sample_record(content={})
@@ -60,7 +68,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
60
68
  reload_after 10
61
69
  catch_transport_exception_on_retry false # For fallback testing
62
70
  ]
63
- assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
71
+ assert_raise(TRANSPORT_CLASS::Transport::Errors::NotFound) do
64
72
  driver(config)
65
73
  end
66
74
  driver.run(default_tag: 'test') do