fluent-plugin-elasticsearch 4.1.1 → 5.4.3

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.1.1'
+  s.version       = '5.4.3'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -23,11 +23,14 @@ Gem::Specification.new do |s|
   s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
 
   s.add_runtime_dependency 'fluentd', '>= 0.14.22'
+  s.add_runtime_dependency 'faraday', '>= 2.0.0'
+  s.add_runtime_dependency 'faraday-excon', '>= 2.0.0'
   s.add_runtime_dependency 'excon', '>= 0'
   s.add_runtime_dependency 'elasticsearch'
 
 
   s.add_development_dependency 'rake', '>= 0'
+  s.add_development_dependency 'webrick', '~> 1.7.0'
   s.add_development_dependency 'webmock', '~> 3'
   s.add_development_dependency 'test-unit', '~> 3.3.0'
   s.add_development_dependency 'minitest', '~> 5.8'

data/lib/fluent/plugin/elasticsearch_compat.rb

@@ -0,0 +1,31 @@
+begin
+  require 'elastic/transport'
+  ::TRANSPORT_CLASS = Elastic::Transport
+rescue LoadError
+end
+begin
+  require 'elasticsearch/transport'
+  ::TRANSPORT_CLASS = Elasticsearch::Transport
+rescue LoadError
+end
+if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("8.0.0")
+  begin
+    require 'elasticsearch/xpack'
+  rescue LoadError
+    require 'elasticsearch/api' # For elasticsearch-ruby 8 or later
+  end
+end
+
+begin
+  require 'elastic/transport/transport/connections/selector'
+  ::SELECTOR_CLASS = Elastic::Transport::Transport::Connections::Selector
+rescue LoadError
+end
+begin
+  require 'elasticsearch/transport/transport/connections/selector'
+  ::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
+rescue LoadError
+end
+unless defined?(::Elasticsearch::UnsupportedProductError)
+  class ::Elasticsearch::UnsupportedProductError < StandardError; end
+end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -23,6 +23,10 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     unrecoverable_error_types.include?(type)
   end
 
+  def unrecoverable_record_error?(type)
+    ['json_parse_exception'].include?(type)
+  end
+
   def log_es_400_reason(&block)
     if @plugin.log_es_400_reason
       block.call
@@ -31,7 +35,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     end
   end
 
-  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+  def handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
     items = response['items']
     if items.nil? || !items.is_a?(Array)
       raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
@@ -43,15 +47,21 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     stats = Hash.new(0)
     meta = {}
     header = {}
-    chunk.msgpack_each do |time, rawrecord|
+    affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
+
+    unpacked_msg_arr.each do |msg|
+      time = msg[:time]
+      rawrecord = msg[:record]
+
       bulk_message = ''
       next unless rawrecord.is_a? Hash
       begin
         # we need a deep copy for process_message to alter
         processrecord = Marshal.load(Marshal.dump(rawrecord))
-        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
+        meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
         next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
       rescue => e
+        @plugin.log.debug("Exception in error handler during deep copy: #{e}")
         stats[:bad_chunk_record] += 1
         next
       end
@@ -105,10 +115,15 @@ class Fluent::Plugin::ElasticsearchErrorHandler
         elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
           type = item[write_operation]['error']['type']
           stats[type] += 1
-          retry_stream.add(time, rawrecord)
           if unrecoverable_error?(type)
             raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
           end
+          if unrecoverable_record_error?(type)
+            @plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - #{type}: #{reason}"))
+            next
+          else
+            retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
+          end
         else
           # When we don't have a type field, something changed in the API
           # expected return values (ES 2.x)

data/lib/fluent/plugin/elasticsearch_fallback_selector.rb

@@ -1,7 +1,7 @@
-require 'elasticsearch/transport/transport/connections/selector'
+require_relative 'elasticsearch_compat'
 
 class Fluent::Plugin::ElasticseatchFallbackSelector
-  include Elasticsearch::Transport::Transport::Connections::Selector::Base
+  include SELECTOR_CLASS::Base
 
   def select(options={})
     connections.first

data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb

@@ -1,3 +1,5 @@
+require_relative 'elasticsearch_compat'
+
 module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
   ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
 
@@ -21,7 +23,7 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
       raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
       raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
 
-    rescue Elasticsearch::Transport::Transport::Error => e
+    rescue ::TRANSPORT_CLASS::Transport::Error => e
       raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
     end
   end
@@ -43,12 +45,20 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
   end
 
   def get_ilm_policy
-    client.ilm.get_policy
+    if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+      client.ilm.get_lifecycle
+    else
+      client.ilm.get_policy
+    end
   end
 
   def ilm_policy_exists?(policy_id)
     begin
-      client.ilm.get_policy(policy_id: policy_id)
+      if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+        client.ilm.get_lifecycle(policy: policy_id)
+      else
+        client.ilm.get_policy(policy_id: policy_id)
+      end
       true
     rescue
       false
@@ -57,7 +67,11 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
 
   def ilm_policy_put(policy_id, policy)
     log.info("Installing ILM policy: #{policy}")
-    client.ilm.put_policy(policy_id: policy_id, body: policy)
+    if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+      client.ilm.put_lifecycle(policy: policy_id, body: policy)
+    else
+      client.ilm.put_policy(policy_id: policy_id, body: policy)
+    end
   end
 
   def default_policy_payload

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -3,7 +3,7 @@ require_relative './elasticsearch_error'
 
 module Fluent::ElasticsearchIndexTemplate
   def get_template(template_file)
-    if !File.exists?(template_file)
+    if !File.exist?(template_file)
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')
@@ -11,7 +11,7 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def get_custom_template(template_file, customize_template)
-    if !File.exists?(template_file)
+    if !File.exist?(template_file)
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
     end
     file_contents = IO.read(template_file).gsub(/\n/,'')
@@ -22,18 +22,39 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_exists?(name, host = nil)
-    client(host).indices.get_template(:name => name)
+    if @use_legacy_template
+      client(host).indices.get_template(:name => name)
+    else
+      client(host).indices.get_index_template(:name => name)
+    end
     return true
-  rescue Elasticsearch::Transport::Transport::Errors::NotFound
+  rescue TRANSPORT_CLASS::Transport::Errors::NotFound
     return false
   end
 
-  def retry_operate(max_retries, fail_on_retry_exceed = true)
+  def host_unreachable_exceptions
+    if Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      # elasticsearch-ruby 8.0.0's elastic-transport uses
+      # direct callable #host_unreachable_exceptions again.
+      client.transport.host_unreachable_exceptions
+    elsif Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      # elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
+      # Elasticsearch class on Transport.
+      # This is why #host_unreachable_exceptions is not callable directly
+      # via transport (not transport's transport instance accessor) any more.
+      client.transport.transport.host_unreachable_exceptions
+    else
+      client.transport.host_unreachable_exceptions
+    end
+  end
+
+  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
+    transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map{ |c| TRANSPORT_CLASS::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
-    rescue *client.transport.host_unreachable_exceptions, Timeout::Error => e
+    rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries
@@ -52,27 +73,32 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_put(name, template, host = nil)
-    client(host).indices.put_template(:name => name, :body => template)
+    if @use_legacy_template
+      client(host).indices.put_template(:name => name, :body => template)
+    else
+      client(host).indices.put_index_template(:name => name, :body => template)
+    end
   end
 
   def indexcreation(index_name, host = nil)
     client(host).indices.create(:index => index_name)
-  rescue Elasticsearch::Transport::Transport::Error => e
+  rescue TRANSPORT_CLASS::Transport::Error => e
     if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else
-      log.error("Error while index creation - #{index_name}: #{e.inspect}")
+      log.error("Error while index creation - #{index_name}", error: e)
     end
   end
 
-  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
     inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
     if overwrite
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
 
       log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
@@ -83,7 +109,8 @@ module Fluent::ElasticsearchIndexTemplate
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
       log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
     else
@@ -91,14 +118,15 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
     template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
     custom_template = if enable_ilm
                         inject_ilm_settings_to_template(deflector_alias_name,
                                                         target_index,
                                                         ilm_policy_id,
                                                         get_custom_template(template_file,
-                                                                            customize_template))
+                                                                            customize_template),
+                                                        index_separator)
                       else
                         get_custom_template(template_file, customize_template)
                       end
@@ -119,15 +147,31 @@ module Fluent::ElasticsearchIndexTemplate
     enable_ilm ? deflector_alias_name : template_name
   end
 
-  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
+  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
     log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
-    template.delete('template') if template.include?('template')
-    template['index_patterns'] = "#{target_index}-*"
-    template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
-    if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
-      log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+    template['index_patterns'] = "#{target_index}#{index_separator}*"
+    if @use_legacy_template
+      template.delete('template') if template.include?('template')
+      # Prepare settings Hash
+      if !template.key?('settings')
+        template['settings'] = {}
+      end
+      if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
+    else
+      # Prepare template.settings Hash
+      if !template['template'].key?('settings')
+        template['template']['settings'] = {}
+      end
+      if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
     end
-    template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
     template
   end
 

data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb

@@ -1,6 +1,7 @@
 require 'elasticsearch'
+require_relative 'elasticsearch_compat'
 
-class Fluent::Plugin::ElasticsearchSimpleSniffer < Elasticsearch::Transport::Transport::Sniffer
+class Fluent::Plugin::ElasticsearchSimpleSniffer < TRANSPORT_CLASS::Transport::Sniffer
 
   def hosts
     @transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -53,7 +53,7 @@ module Fluent::Plugin
       seed += tag + separator if @include_tag_in_seed
       seed += time.to_s + separator if @include_time_in_seed
       if @use_entire_record
-        record.each {|k,v| seed += "|#{k}|#{v}"}
+        record.keys.sort.each {|k| seed += "|#{k}|#{record[k]}"}
       else
         seed += record_keys.map {|k| record[k]}.join(separator)
       end

data/lib/fluent/plugin/in_elasticsearch.rb

@@ -1,8 +1,10 @@
 require 'elasticsearch'
 
+require 'faraday/excon'
 require 'fluent/log-ext'
 require 'fluent/plugin/input'
 require_relative 'elasticsearch_constants'
+require_relative 'elasticsearch_compat'
 
 module Fluent::Plugin
   class ElasticsearchInput < Input
@@ -218,7 +220,7 @@ module Fluent::Plugin
 
         headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
 
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(
+        transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(
           connection_options.merge(
             options: {
               reload_connections: local_reload_connections,
@@ -284,7 +286,11 @@ module Fluent::Plugin
       end
 
       router.emit_stream(@tag, es)
-      client.clear_scroll(scroll_id: scroll_id) if scroll_id
+      if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("7.0.0")
+        client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
+      else
+        client.clear_scroll(scroll_id: scroll_id) if scroll_id
+      end
     end
 
     def process_scroll_request(scroll_id)

data/lib/fluent/plugin/oj_serializer.rb

@@ -1,10 +1,11 @@
 require 'oj'
+require_relative 'elasticsearch_compat'
 
 module Fluent::Plugin
   module Serializer
 
     class Oj
-      include Elasticsearch::Transport::Transport::Serializer::Base
+      include TRANSPORT_CLASS::Transport::Serializer::Base
 
       # De-serialize a Hash from JSON string
       #