fluent-plugin-elasticsearch 4.0.5 → 4.0.10

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.5'
+  s.version       = '4.0.10'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/gemfiles/Gemfile.elasticsearch.v6

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 gemspec :path => "../"
 
 gem 'simplecov', require: false
-gem 'coveralls', require: false
+gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
 gem "elasticsearch", "~> 6.8.1"

data/gemfiles/{Gemfile.ilm → Gemfile.without.ilm}

@@ -4,8 +4,7 @@ source 'https://rubygems.org'
 gemspec :path => "../"
 
 gem 'simplecov', require: false
-gem 'coveralls', require: false
+gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
-gem "elasticsearch-xpack"
 gem "oj"

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -58,7 +58,7 @@ module Fluent::ElasticsearchIndexTemplate
   def indexcreation(index_name, host = nil)
     client(host).indices.create(:index => index_name)
   rescue Elasticsearch::Transport::Transport::Error => e
-    if e.message =~ /"already exists"/
+    if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else
       log.error("Error while index creation - #{index_name}: #{e.inspect}")

data/lib/fluent/plugin/elasticsearch_tls.rb

@@ -10,7 +10,7 @@ module Fluent::Plugin
                                [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
                              end
 
-    DEFAULT_VERSION = :TLSv1
+    DEFAULT_VERSION = :TLSv1_2
     METHODS_MAP = begin
                     # When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
                     map = {
@@ -48,8 +48,8 @@ module Fluent::Plugin
       if USE_TLS_MINMAX_VERSION
         case
         when ssl_min_version.nil? && ssl_max_version.nil?
-          ssl_min_version = METHODS_MAP[ssl_version]
-          ssl_max_version = METHODS_MAP[ssl_version]
+          ssl_min_version = METHODS_MAP[:TLSv1_2]
+          ssl_max_version = METHODS_MAP[:TLSv1_3]
         when ssl_min_version && ssl_max_version.nil?
           raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
         when ssl_min_version.nil? && ssl_max_version

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -7,6 +7,13 @@ module Fluent::Plugin
     Fluent::Plugin.register_filter('elasticsearch_genid', self)
 
     config_param :hash_id_key, :string, :default => '_hash'
+    config_param :include_tag_in_seed, :bool, :default => false
+    config_param :include_time_in_seed, :bool, :default => false
+    config_param :use_record_as_seed, :bool, :default => false
+    config_param :use_entire_record, :bool, :default => false
+    config_param :record_keys, :array, :default => []
+    config_param :separator, :string, :default => '_'
+    config_param :hash_type, :enum, list: [:md5, :sha1, :sha256, :sha512], :default => :sha1
 
     def initialize
       super
@@ -14,12 +21,57 @@ module Fluent::Plugin
 
     def configure(conf)
       super
+
+      if !@use_entire_record
+        if @record_keys.empty? && @use_record_as_seed
+          raise Fluent::ConfigError, "When using record as hash seed, users must specify `record_keys`."
+        end
+      end
+
+      if @use_record_as_seed
+        class << self
+          alias_method :filter, :filter_seed_as_record
+        end
+      else
+        class << self
+          alias_method :filter, :filter_simple
+        end
+      end
     end
 
     def filter(tag, time, record)
+      # for safety.
+    end
+
+    def filter_simple(tag, time, record)
       record[@hash_id_key] = Base64.strict_encode64(SecureRandom.uuid)
       record
     end
 
+    def filter_seed_as_record(tag, time, record)
+      seed = ""
+      seed += tag + separator if @include_tag_in_seed
+      seed += time.to_s + separator if @include_time_in_seed
+      if @use_entire_record
+        record.each {|k,v| seed += "|#{k}|#{v}"}
+      else
+        seed += record_keys.map {|k| record[k]}.join(separator)
+      end
+      record[@hash_id_key] = Base64.strict_encode64(encode_hash(@hash_type, seed))
+      record
+    end
+
+    def encode_hash(type, seed)
+      case type
+      when :md5
+        Digest::MD5.digest(seed)
+      when :sha1
+        Digest::SHA1.digest(seed)
+      when :sha256
+        Digest::SHA256.digest(seed)
+      when :sha512
+        Digest::SHA512.digest(seed)
+      end
+    end
   end
 end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -55,6 +55,7 @@ module Fluent::Plugin
     attr_reader :alias_indexes
     attr_reader :template_names
     attr_reader :ssl_version_options
+    attr_reader :compressable_connection
 
     helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
@@ -89,6 +90,7 @@ EOC
     config_param :logstash_dateformat, :string, :default => "%Y.%m.%d"
     config_param :utc_index, :bool, :default => true
     config_param :type_name, :string, :default => DEFAULT_TYPE_NAME
+    config_param :suppress_type_name, :bool, :default => false
     config_param :index_name, :string, :default => "fluentd"
     config_param :id_key, :string, :default => nil
     config_param :write_operation, :string, :default => "index"
@@ -125,6 +127,7 @@ EOC
     config_param :templates, :hash, :default => nil
     config_param :max_retry_putting_template, :integer, :default => 10
     config_param :fail_on_putting_template_retry_exceed, :bool, :default => true
+    config_param :fail_on_detecting_es_version_retry_exceed, :bool, :default => true
     config_param :max_retry_get_es_version, :integer, :default => 15
     config_param :include_tag_key, :bool, :default => false
     config_param :tag_key, :string, :default => 'tag'
@@ -142,6 +145,7 @@ see: https://github.com/elastic/elasticsearch-ruby/pull/514
 EOC
     config_param :include_index_in_url, :bool, :default => false
     config_param :http_backend, :enum, list: [:excon, :typhoeus], :default => :excon
+    config_param :http_backend_excon_nonblock, :bool, :default => true
     config_param :validate_client_version, :bool, :default => false
     config_param :prefer_oj_serializer, :bool, :default => false
     config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "es_rejected_execution_exception"]
@@ -157,6 +161,7 @@ EOC
     config_param :enable_ilm, :bool, :default => false
     config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
     config_param :ilm_policy, :hash, :default => {}
+    config_param :ilm_policies, :hash, :default => {}
     config_param :ilm_policy_overwrite, :bool, :default => false
     config_param :truncate_caches_interval, :time, :default => nil
 
@@ -217,6 +222,11 @@ EOC
         log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
       end
 
+      raise Fluent::ConfigError, "You can't specify ilm_policy and ilm_policies at the same time" unless @ilm_policy.empty? or @ilm_policies.empty?
+
+      unless @ilm_policy.empty?
+        @ilm_policies = { @ilm_policy_id => @ilm_policy }
+      end
       @alias_indexes = []
       @template_names = []
       if !dry_run?
@@ -225,14 +235,14 @@ EOC
             raise Fluent::ConfigError, "deflector_alias is prohibited to use with 'logstash_format at same time." if @logstash_format and @deflector_alias
           end
           if @ilm_policy.empty? && @ilm_policy_overwrite
-            raise Fluent::ConfigError, "ilm_policy_overwrite can work with non empty ilm_policy. Specify non-empty ilm policy into ilm_policy. "
+            raise Fluent::ConfigError, "ilm_policy_overwrite requires a non empty ilm_policy."
           end
           if @logstash_format || placeholder_substitution_needed_for_template?
             class << self
               alias_method :template_installation, :template_installation_actual
             end
           else
-            template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
+            template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
           end
           verify_ilm_working if @enable_ilm
         elsif @templates
@@ -290,24 +300,25 @@ EOC
         raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
       end
 
-      @last_seen_major_version =
-        if @verify_es_version_at_startup && !dry_run?
-          retry_operate(@max_retry_get_es_version) do
-            detect_es_major_version
-          end
-        else
-          @default_elasticsearch_version
-        end
-      if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
-      end
-      if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
-        @type_name = '_doc'.freeze
-      end
-      if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-        log.info "Detected ES 8.x or above: This parameter has no effect."
+      @last_seen_major_version = if major_version = handle_last_seen_es_major_version
+                                   major_version
+                                 else
+                                   @default_elasticsearch_version
+                                 end
+      if @suppress_type_name && @last_seen_major_version >= 7
         @type_name = nil
+      else
+        if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
+        end
+        if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
+          @type_name = '_doc'.freeze
+        end
+        if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.info "Detected ES 8.x or above: This parameter has no effect."
+          @type_name = nil
+        end
       end
 
       if @validate_client_version && !dry_run?
@@ -329,6 +340,13 @@ EOC
         end
       end
 
+      if @ssl_version && @scheme == :https
+        if !@http_backend_excon_nonblock
+          log.warn "TLS handshake will be stucked with block connection.
+                    Consider to set `http_backend_excon_nonblock` as true"
+        end
+      end
+
       # Consider missing the prefix of "$." in nested key specifiers.
       @id_key = convert_compat_id_key(@id_key) if @id_key
       @parent_key = convert_compat_id_key(@parent_key) if @parent_key
@@ -337,6 +355,7 @@ EOC
       @routing_key_name = configure_routing_key_name
       @meta_config_map = create_meta_config_map
       @current_config = nil
+      @compressable_connection = false
 
       @ignore_exception_classes = @ignore_exceptions.map do |exception|
         unless Object.const_defined?(exception)
@@ -411,7 +430,7 @@ EOC
     def backend_options
       case @http_backend
       when :excon
-        { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
+        { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
       when :typhoeus
         require 'typhoeus'
         { sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
@@ -421,6 +440,16 @@ EOC
       raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
     end
 
+    def handle_last_seen_es_major_version
+      if @verify_es_version_at_startup && !dry_run?
+        retry_operate(@max_retry_get_es_version, @fail_on_detecting_es_version_retry_exceed) do
+          detect_es_major_version
+        end
+      else
+        nil
+      end
+    end
+
     def detect_es_major_version
       @_es_info ||= client.info
       @_es_info["version"]["number"].to_i
@@ -495,13 +524,15 @@ EOC
       return Time.at(event_time).to_datetime
     end
 
-    def client(host = nil)
+    def client(host = nil, compress_connection = false)
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
       @_es = nil unless is_existing_connection(connection_options[:hosts])
+      @_es = nil unless @compressable_connection == compress_connection
 
       @_es ||= begin
+        @compressable_connection = compress_connection
         @current_config = connection_options[:hosts].clone
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
         local_reload_connections = @reload_connections
@@ -509,7 +540,7 @@ EOC
           local_reload_connections = @reload_after
         end
 
-        gzip_headers = if compression
+        gzip_headers = if compress_connection
                          {'Content-Encoding' => 'gzip'}
                        else
                          {}
@@ -535,7 +566,7 @@ EOC
                                                                               },
                                                                               sniffer_class: @sniffer_class,
                                                                               serializer_class: @serializer_class,
-                                                                              compression: compression,
+                                                                              compression: compress_connection,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport
       end
@@ -699,7 +730,12 @@ EOC
       else
         pipeline = nil
       end
-      return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
+      if @ilm_policy_id
+        ilm_policy_id = extract_placeholders(@ilm_policy_id, chunk)
+      else
+        ilm_policy_id = nil
+      end
+      return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline, ilm_policy_id
     end
 
     def multi_workers_ready?
@@ -773,7 +809,7 @@ EOC
     end
 
     def process_message(tag, meta, header, time, record, extracted_values)
-      logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
+      logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline, _ilm_policy_id = extracted_values
 
       if @flatten_hashes
         record = flatten_record(record)
@@ -825,7 +861,9 @@ EOC
           target_type = nil
         end
       else
-        if @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+        if @suppress_type_name && @last_seen_major_version >= 7
+          target_type = nil
+        elsif @last_seen_major_version == 7 && @type_name != DEFAULT_TYPE_NAME_ES_7x
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           target_type = '_doc'.freeze
         elsif @last_seen_major_version >= 8
@@ -881,16 +919,17 @@ EOC
         placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
         placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
         placeholder?(:deflector_alias, @deflector_alias.to_s) ||
-        placeholder?(:application_name, @application_name.to_s)
+        placeholder?(:application_name, @application_name.to_s) ||
+        placeholder?(:ilm_policy_id, @ilm_policy_id.to_s)
       log.debug("Need substitution: #{need_substitution}")
       need_substitution
     end
 
-    def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
+    def template_installation(deflector_alias, template_name, customize_template, application_name, ilm_policy_id, target_index, host)
       # for safety.
     end
 
-    def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
+    def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
       if template_name && @template_file
         if @alias_indexes.include? deflector_alias
           log.debug("Index alias #{deflector_alias} already exists (cached)")
@@ -899,11 +938,12 @@ EOC
         else
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             if customize_template
-              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, host)
+              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
             else
-              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, host)
+              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
             end
-            create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, @ilm_policy_overwrite, host)
+            ilm_policy = @ilm_policies[ilm_policy_id] || {}
+            create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
           end
           @alias_indexes << deflector_alias unless deflector_alias.nil?
           @template_names << template_name unless template_name.nil?
@@ -914,11 +954,11 @@ EOC
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
     def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
-      logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
+      logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
       if deflector_alias
-        template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
+        template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
       else
-        template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
+        template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
       end
 
       begin
@@ -931,7 +971,7 @@ EOC
                           data
                         end
 
-        response = client(info.host).bulk body: prepared_data, index: info.index
+        response = client(info.host, compression).bulk body: prepared_data, index: info.index
         log.on_trace { log.trace "bulk response: #{response}" }
 
         if response['errors']

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -35,16 +35,18 @@ module Fluent::Plugin
     end
 
 
-    def client(host = nil)
+    def client(host = nil, compress_connection = false)
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
       @_es = nil unless is_existing_connection(connection_options[:hosts])
+      @_es = nil unless @compressable_connection == compress_connection
 
       @_es ||= begin
+        @compressable_connection = compress_connection
         @current_config = connection_options[:hosts].clone
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
-        gzip_headers = if compression
+        gzip_headers = if compress_connection
                          {'Content-Encoding' => 'gzip'}
                        else
                          {}
@@ -67,7 +69,7 @@ module Fluent::Plugin
                                                                                 password: @password,
                                                                                 scheme: @scheme
                                                                               },
-                                                                              compression: compression,
+                                                                              compression: compress_connection,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport
       end
@@ -228,7 +230,7 @@ module Fluent::Plugin
                         else
                           data
                         end
-        response = client(host).bulk body: prepared_data, index: index
+        response = client(host, compression).bulk body: prepared_data, index: index
         if response['errors']
           log.error "Could not push log to Elasticsearch: #{response}"
         end