fluent-plugin-elasticsearch 4.1.1 → 5.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
  4. data/.github/dependabot.yml +6 -0
  5. data/.github/workflows/issue-auto-closer.yml +2 -2
  6. data/.github/workflows/linux.yml +5 -2
  7. data/.github/workflows/macos.yml +5 -2
  8. data/.github/workflows/windows.yml +5 -2
  9. data/Gemfile +1 -2
  10. data/History.md +146 -0
  11. data/README.ElasticsearchGenID.md +4 -4
  12. data/README.ElasticsearchInput.md +1 -1
  13. data/README.Troubleshooting.md +692 -0
  14. data/README.md +260 -550
  15. data/fluent-plugin-elasticsearch.gemspec +4 -1
  16. data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
  17. data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
  18. data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
  19. data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
  20. data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
  21. data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
  22. data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
  23. data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
  24. data/lib/fluent/plugin/oj_serializer.rb +2 -1
  25. data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
  26. data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
  27. data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
  28. data/test/helper.rb +0 -4
  29. data/test/plugin/mock_chunk.dat +0 -0
  30. data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
  31. data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
  32. data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
  33. data/test/plugin/test_elasticsearch_tls.rb +8 -2
  34. data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
  35. data/test/plugin/test_in_elasticsearch.rb +51 -21
  36. data/test/plugin/test_index_alias_template.json +11 -0
  37. data/test/plugin/test_index_template.json +25 -0
  38. data/test/plugin/test_out_elasticsearch.rb +2118 -704
  39. data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
  40. data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
  41. metadata +62 -10
  42. data/.coveralls.yml +0 -2
  43. data/.travis.yml +0 -44
  44. data/appveyor.yml +0 -20
  45. data/gemfiles/Gemfile.without.ilm +0 -10
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.1.1'
6
+ s.version = '5.4.3'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -23,11 +23,14 @@ Gem::Specification.new do |s|
23
23
  s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
24
24
 
25
25
  s.add_runtime_dependency 'fluentd', '>= 0.14.22'
26
+ s.add_runtime_dependency 'faraday', '>= 2.0.0'
27
+ s.add_runtime_dependency 'faraday-excon', '>= 2.0.0'
26
28
  s.add_runtime_dependency 'excon', '>= 0'
27
29
  s.add_runtime_dependency 'elasticsearch'
28
30
 
29
31
 
30
32
  s.add_development_dependency 'rake', '>= 0'
33
+ s.add_development_dependency 'webrick', '~> 1.7.0'
31
34
  s.add_development_dependency 'webmock', '~> 3'
32
35
  s.add_development_dependency 'test-unit', '~> 3.3.0'
33
36
  s.add_development_dependency 'minitest', '~> 5.8'
@@ -0,0 +1,31 @@
1
+ begin
2
+ require 'elastic/transport'
3
+ ::TRANSPORT_CLASS = Elastic::Transport
4
+ rescue LoadError
5
+ end
6
+ begin
7
+ require 'elasticsearch/transport'
8
+ ::TRANSPORT_CLASS = Elasticsearch::Transport
9
+ rescue LoadError
10
+ end
11
+ if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("8.0.0")
12
+ begin
13
+ require 'elasticsearch/xpack'
14
+ rescue LoadError
15
+ require 'elasticsearch/api' # For elasticsearch-ruby 8 or later
16
+ end
17
+ end
18
+
19
+ begin
20
+ require 'elastic/transport/transport/connections/selector'
21
+ ::SELECTOR_CLASS = Elastic::Transport::Transport::Connections::Selector
22
+ rescue LoadError
23
+ end
24
+ begin
25
+ require 'elasticsearch/transport/transport/connections/selector'
26
+ ::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
27
+ rescue LoadError
28
+ end
29
+ unless defined?(::Elasticsearch::UnsupportedProductError)
30
+ class ::Elasticsearch::UnsupportedProductError < StandardError; end
31
+ end
@@ -23,6 +23,10 @@ class Fluent::Plugin::ElasticsearchErrorHandler
23
23
  unrecoverable_error_types.include?(type)
24
24
  end
25
25
 
26
+ def unrecoverable_record_error?(type)
27
+ ['json_parse_exception'].include?(type)
28
+ end
29
+
26
30
  def log_es_400_reason(&block)
27
31
  if @plugin.log_es_400_reason
28
32
  block.call
@@ -31,7 +35,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
31
35
  end
32
36
  end
33
37
 
34
- def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
38
+ def handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
35
39
  items = response['items']
36
40
  if items.nil? || !items.is_a?(Array)
37
41
  raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
@@ -43,15 +47,21 @@ class Fluent::Plugin::ElasticsearchErrorHandler
43
47
  stats = Hash.new(0)
44
48
  meta = {}
45
49
  header = {}
46
- chunk.msgpack_each do |time, rawrecord|
50
+ affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
51
+
52
+ unpacked_msg_arr.each do |msg|
53
+ time = msg[:time]
54
+ rawrecord = msg[:record]
55
+
47
56
  bulk_message = ''
48
57
  next unless rawrecord.is_a? Hash
49
58
  begin
50
59
  # we need a deep copy for process_message to alter
51
60
  processrecord = Marshal.load(Marshal.dump(rawrecord))
52
- meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
61
+ meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
53
62
  next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
54
63
  rescue => e
64
+ @plugin.log.debug("Exception in error handler during deep copy: #{e}")
55
65
  stats[:bad_chunk_record] += 1
56
66
  next
57
67
  end
@@ -105,10 +115,15 @@ class Fluent::Plugin::ElasticsearchErrorHandler
105
115
  elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
106
116
  type = item[write_operation]['error']['type']
107
117
  stats[type] += 1
108
- retry_stream.add(time, rawrecord)
109
118
  if unrecoverable_error?(type)
110
119
  raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
111
120
  end
121
+ if unrecoverable_record_error?(type)
122
+ @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - #{type}: #{reason}"))
123
+ next
124
+ else
125
+ retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
126
+ end
112
127
  else
113
128
  # When we don't have a type field, something changed in the API
114
129
  # expected return values (ES 2.x)
@@ -1,7 +1,7 @@
1
- require 'elasticsearch/transport/transport/connections/selector'
1
+ require_relative 'elasticsearch_compat'
2
2
 
3
3
  class Fluent::Plugin::ElasticseatchFallbackSelector
4
- include Elasticsearch::Transport::Transport::Connections::Selector::Base
4
+ include SELECTOR_CLASS::Base
5
5
 
6
6
  def select(options={})
7
7
  connections.first
@@ -1,3 +1,5 @@
1
+ require_relative 'elasticsearch_compat'
2
+
1
3
  module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
2
4
  ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
3
5
 
@@ -21,7 +23,7 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
21
23
  raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
22
24
  raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
23
25
 
24
- rescue Elasticsearch::Transport::Transport::Error => e
26
+ rescue ::TRANSPORT_CLASS::Transport::Error => e
25
27
  raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
26
28
  end
27
29
  end
@@ -43,12 +45,20 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
43
45
  end
44
46
 
45
47
  def get_ilm_policy
46
- client.ilm.get_policy
48
+ if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
49
+ client.ilm.get_lifecycle
50
+ else
51
+ client.ilm.get_policy
52
+ end
47
53
  end
48
54
 
49
55
  def ilm_policy_exists?(policy_id)
50
56
  begin
51
- client.ilm.get_policy(policy_id: policy_id)
57
+ if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
58
+ client.ilm.get_lifecycle(policy: policy_id)
59
+ else
60
+ client.ilm.get_policy(policy_id: policy_id)
61
+ end
52
62
  true
53
63
  rescue
54
64
  false
@@ -57,7 +67,11 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
57
67
 
58
68
  def ilm_policy_put(policy_id, policy)
59
69
  log.info("Installing ILM policy: #{policy}")
60
- client.ilm.put_policy(policy_id: policy_id, body: policy)
70
+ if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
71
+ client.ilm.put_lifecycle(policy: policy_id, body: policy)
72
+ else
73
+ client.ilm.put_policy(policy_id: policy_id, body: policy)
74
+ end
61
75
  end
62
76
 
63
77
  def default_policy_payload
@@ -3,7 +3,7 @@ require_relative './elasticsearch_error'
3
3
 
4
4
  module Fluent::ElasticsearchIndexTemplate
5
5
  def get_template(template_file)
6
- if !File.exists?(template_file)
6
+ if !File.exist?(template_file)
7
7
  raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
8
8
  end
9
9
  file_contents = IO.read(template_file).gsub(/\n/,'')
@@ -11,7 +11,7 @@ module Fluent::ElasticsearchIndexTemplate
11
11
  end
12
12
 
13
13
  def get_custom_template(template_file, customize_template)
14
- if !File.exists?(template_file)
14
+ if !File.exist?(template_file)
15
15
  raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
16
16
  end
17
17
  file_contents = IO.read(template_file).gsub(/\n/,'')
@@ -22,18 +22,39 @@ module Fluent::ElasticsearchIndexTemplate
22
22
  end
23
23
 
24
24
  def template_exists?(name, host = nil)
25
- client(host).indices.get_template(:name => name)
25
+ if @use_legacy_template
26
+ client(host).indices.get_template(:name => name)
27
+ else
28
+ client(host).indices.get_index_template(:name => name)
29
+ end
26
30
  return true
27
- rescue Elasticsearch::Transport::Transport::Errors::NotFound
31
+ rescue TRANSPORT_CLASS::Transport::Errors::NotFound
28
32
  return false
29
33
  end
30
34
 
31
- def retry_operate(max_retries, fail_on_retry_exceed = true)
35
+ def host_unreachable_exceptions
36
+ if Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
37
+ # elasticsearch-ruby 8.0.0's elastic-transport uses
38
+ # direct callable #host_unreachable_exceptions again.
39
+ client.transport.host_unreachable_exceptions
40
+ elsif Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
41
+ # elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
42
+ # Elasticsearch class on Transport.
43
+ # This is why #host_unreachable_exceptions is not callable directly
44
+ # via transport (not transport's transport instance accessor) any more.
45
+ client.transport.transport.host_unreachable_exceptions
46
+ else
47
+ client.transport.host_unreachable_exceptions
48
+ end
49
+ end
50
+
51
+ def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
32
52
  return unless block_given?
33
53
  retries = 0
54
+ transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map{ |c| TRANSPORT_CLASS::Transport::Errors.const_get c } if catch_trasport_exceptions
34
55
  begin
35
56
  yield
36
- rescue *client.transport.host_unreachable_exceptions, Timeout::Error => e
57
+ rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
37
58
  @_es = nil
38
59
  @_es_info = nil
39
60
  if retries < max_retries
@@ -52,27 +73,32 @@ module Fluent::ElasticsearchIndexTemplate
52
73
  end
53
74
 
54
75
  def template_put(name, template, host = nil)
55
- client(host).indices.put_template(:name => name, :body => template)
76
+ if @use_legacy_template
77
+ client(host).indices.put_template(:name => name, :body => template)
78
+ else
79
+ client(host).indices.put_index_template(:name => name, :body => template)
80
+ end
56
81
  end
57
82
 
58
83
  def indexcreation(index_name, host = nil)
59
84
  client(host).indices.create(:index => index_name)
60
- rescue Elasticsearch::Transport::Transport::Error => e
85
+ rescue TRANSPORT_CLASS::Transport::Error => e
61
86
  if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
62
87
  log.debug("Index #{index_name} already exists")
63
88
  else
64
- log.error("Error while index creation - #{index_name}: #{e.inspect}")
89
+ log.error("Error while index creation - #{index_name}", error: e)
65
90
  end
66
91
  end
67
92
 
68
- def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
93
+ def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
69
94
  inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
70
95
  if overwrite
71
96
  template_put(inject_template_name,
72
97
  enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
73
98
  target_index,
74
99
  ilm_policy_id,
75
- get_template(template_file)) :
100
+ get_template(template_file),
101
+ index_separator) :
76
102
  get_template(template_file), host)
77
103
 
78
104
  log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
@@ -83,7 +109,8 @@ module Fluent::ElasticsearchIndexTemplate
83
109
  enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
84
110
  target_index,
85
111
  ilm_policy_id,
86
- get_template(template_file)) :
112
+ get_template(template_file),
113
+ index_separator) :
87
114
  get_template(template_file), host)
88
115
  log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
89
116
  else
@@ -91,14 +118,15 @@ module Fluent::ElasticsearchIndexTemplate
91
118
  end
92
119
  end
93
120
 
94
- def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
121
+ def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
95
122
  template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
96
123
  custom_template = if enable_ilm
97
124
  inject_ilm_settings_to_template(deflector_alias_name,
98
125
  target_index,
99
126
  ilm_policy_id,
100
127
  get_custom_template(template_file,
101
- customize_template))
128
+ customize_template),
129
+ index_separator)
102
130
  else
103
131
  get_custom_template(template_file, customize_template)
104
132
  end
@@ -119,15 +147,31 @@ module Fluent::ElasticsearchIndexTemplate
119
147
  enable_ilm ? deflector_alias_name : template_name
120
148
  end
121
149
 
122
- def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
150
+ def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
123
151
  log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
124
- template.delete('template') if template.include?('template')
125
- template['index_patterns'] = "#{target_index}-*"
126
- template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
127
- if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
128
- log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
152
+ template['index_patterns'] = "#{target_index}#{index_separator}*"
153
+ if @use_legacy_template
154
+ template.delete('template') if template.include?('template')
155
+ # Prepare settings Hash
156
+ if !template.key?('settings')
157
+ template['settings'] = {}
158
+ end
159
+ if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
160
+ log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
161
+ end
162
+ template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
163
+ template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
164
+ else
165
+ # Prepare template.settings Hash
166
+ if !template['template'].key?('settings')
167
+ template['template']['settings'] = {}
168
+ end
169
+ if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
170
+ log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
171
+ end
172
+ template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
173
+ template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
129
174
  end
130
- template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
131
175
  template
132
176
  end
133
177
 
@@ -1,6 +1,7 @@
1
1
  require 'elasticsearch'
2
+ require_relative 'elasticsearch_compat'
2
3
 
3
- class Fluent::Plugin::ElasticsearchSimpleSniffer < Elasticsearch::Transport::Transport::Sniffer
4
+ class Fluent::Plugin::ElasticsearchSimpleSniffer < TRANSPORT_CLASS::Transport::Sniffer
4
5
 
5
6
  def hosts
6
7
  @transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
@@ -53,7 +53,7 @@ module Fluent::Plugin
53
53
  seed += tag + separator if @include_tag_in_seed
54
54
  seed += time.to_s + separator if @include_time_in_seed
55
55
  if @use_entire_record
56
- record.each {|k,v| seed += "|#{k}|#{v}"}
56
+ record.keys.sort.each {|k| seed += "|#{k}|#{record[k]}"}
57
57
  else
58
58
  seed += record_keys.map {|k| record[k]}.join(separator)
59
59
  end
@@ -1,8 +1,10 @@
1
1
  require 'elasticsearch'
2
2
 
3
+ require 'faraday/excon'
3
4
  require 'fluent/log-ext'
4
5
  require 'fluent/plugin/input'
5
6
  require_relative 'elasticsearch_constants'
7
+ require_relative 'elasticsearch_compat'
6
8
 
7
9
  module Fluent::Plugin
8
10
  class ElasticsearchInput < Input
@@ -218,7 +220,7 @@ module Fluent::Plugin
218
220
 
219
221
  headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
220
222
 
221
- transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(
223
+ transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(
222
224
  connection_options.merge(
223
225
  options: {
224
226
  reload_connections: local_reload_connections,
@@ -284,7 +286,11 @@ module Fluent::Plugin
284
286
  end
285
287
 
286
288
  router.emit_stream(@tag, es)
287
- client.clear_scroll(scroll_id: scroll_id) if scroll_id
289
+ if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("7.0.0")
290
+ client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
291
+ else
292
+ client.clear_scroll(scroll_id: scroll_id) if scroll_id
293
+ end
288
294
  end
289
295
 
290
296
  def process_scroll_request(scroll_id)
@@ -1,10 +1,11 @@
1
1
  require 'oj'
2
+ require_relative 'elasticsearch_compat'
2
3
 
3
4
  module Fluent::Plugin
4
5
  module Serializer
5
6
 
6
7
  class Oj
7
- include Elasticsearch::Transport::Transport::Serializer::Base
8
+ include TRANSPORT_CLASS::Transport::Serializer::Base
8
9
 
9
10
  # De-serialize a Hash from JSON string
10
11
  #