fluent-plugin-elasticsearch 4.1.2 → 4.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 367f610faf1266994e676a07ca2fc391bf0951036f356fd4a95a6a7d7cce1b61
-  data.tar.gz: 56fc5e47f6dd8e63df03f6190fc0f3dfff2e1fae548b38cc9cdb328f75246ade
+  metadata.gz: d54b3f207af54f6fb5d3119f2d6d649478e25471fda03c56e00f9206ae72914e
+  data.tar.gz: 82eea8ed3938919f6860582698ce1253214531c806c132723e8845adfb45e2e0
 SHA512:
-  metadata.gz: 812b336dcf870a4dbb2ecfbc87304b8c6b6cfafb561d2cf6eece11f35e29e9aa484ab8365fa9245a578c51ec540fcaf09925dc68668820ce3ab7c71041720dd2
-  data.tar.gz: da2c1b8bbbdd364ff5a3df1214a54151a7e9e809230b12e75e06b42a0eb7df790124898cd6c137c82edede8c94f5e8adc9d7eea05134bff0a084d1de11b5269e
+  metadata.gz: c4fd2785f716d97cb8c44845c51f933f7d0ee69fe3e2911f85bf621d043015e815639350e363ce42866b224905fc20ae12b3666d302e5f1f07937f047123b20d
+  data.tar.gz: 15d34dc457c5a98b77f2142b271d3f4ef5f611e38caeb4f225f12dcfc2c3771dea0153bae3ebe99a83cc71c6e414d67e39fea1d2fa1fa430049f90839c677578

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,6 +2,30 @@
 
 ### [Unreleased]
 
+### 4.2.2
+- Remove unnecessary nil check (#826)
+- Fix ILM rollover index template pattern to apply index_separator (#825)
+- Fix ILM rollover alias creation when a placeholder is used in index_name (#823)
+- Add a note about the pitfalls of per-date indexes used with ILM (#822)
+
+### 4.2.1
+- Update a broken link (#821)
+- Include chunk_id in records (#820)
+- Fix a failing testcase (#818)
+
+### 4.2.0
+- ci: Add Ruby 2.7 jobs (#812)
+- Support Elasticsearch new style template (#810)
+
+### 4.1.4
+- ElasticsearchGenID update docs for hash_type (#809)
+- Handle api key header (#808)
+- index_template: Handle error object entirely on index creation failure (#807)
+
+### 4.1.3
+- Load multiple templates even if template_name and template_file given (#799)
+- Handle elasticsearch-ruby 7.9.0 using HTTP method changes (#795)
+
 ### 4.1.2
 - Use Hash#dig instead of Hash#[] on retrieving version information from Elasticsearch info API (#793)
 

data/README.ElasticsearchGenID.md

@@ -26,7 +26,7 @@ In your Fluentd configuration, use `@type elasticsearch_genid`. Additional confi
   use_entire_record false
   record_keys []
   separator _
-  hash_type md5
+  hash_type sha1
 </match>
 ```
 
@@ -95,7 +95,7 @@ You can specify separator charactor to creating seed for hash generation.
 hash_type sha1
 ```
 
-You can specify hash algorithm.
+You can specify hash algorithm. Support algorithms `md5`, `sha1`, `sha256`, `sha512`. Default: `sha1`
 
 ## Advanced Usage
 

data/README.md

@@ -85,6 +85,7 @@ Current maintainers: @cosmo0920
   + [verify_es version at startup](#verify_es_version_at_startup)
   + [default_elasticsearch_version](#default_elasticsearch_version)
   + [custom_headers](#custom_headers)
+  + [api_key](#api_key)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
   + [Placeholders](#placeholders)
@@ -100,6 +101,10 @@ Current maintainers: @cosmo0920
   + [ilm_policies](#ilm_policies)
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
   + [truncate_caches_interval](#truncate_caches_interval)
+  + [use_legacy_template](#use_legacy_template)
+  + [metadata section](#metadata-section)
+    + [include_chunk_id](#include_chunk_id)
+    + [chunk_id_key](#chunk_id_key)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
 * [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
 * [Elasticsearch permissions](#elasticsearch-permissions)
@@ -431,7 +436,7 @@ Specify index templates in form of hash. Can contain multiple templates.
 templates { "template_name_1": "path_to_template_1_file", "template_name_2": "path_to_template_2_file"}
 ```
 
-If `template_file` and `template_name` are set, then this parameter will be ignored.
+**Note:** Before ES plugin v4.1.2, if `template_file` and `template_name` are set, then this parameter will be ignored. In 4.1.3 or later, `template_file` and `template_name` can work with `templates`.
 
 ### customize_template
 
@@ -919,7 +924,7 @@ Starting with version 0.8.0, this gem uses excon, which supports proxy with envi
 
 ### Buffer options
 
-`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/v0.14/articles/output-plugin-overview) and use `compat_parameters` plugin helper. It adds the following options:
+`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/output#overview) and use `compat_parameters` plugin helper. It adds the following options:
 
 ```
 buffer_type memory
@@ -1106,6 +1111,14 @@ This parameter adds additional headers to request. The default value is `{}`.
 custom_headers {"token":"secret"}
 ```
 
+### api_key
+
+This parameter adds authentication header. The default value is `nil`.
+
+```
+api_key "ElasticsearchAPIKEY"
+```
+
 ### Not seeing a config you need?
 
 We try to keep the scope of this plugin small and not add too many configuration options. If you think an option would be useful to others, feel free to open an issue or contribute a Pull Request.
@@ -1294,6 +1307,63 @@ If it is set, timer for clearing `alias_indexes` and `template_names` caches wil
 
 Default value is `nil`.
 
+## use_legacy_template
+
+Use legacy template or not.
+
+Elasticsearch 7.8 or later supports the brand new composable templates.
+
+For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
+
+Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
+
+Please confirm that whether the using Elasticsearch cluster(s) support the composable template feature or not when turn on the brand new feature with this parameter.
+
+## <metadata\> section
+
+Users can specify whether including `chunk_id` information into records or not:
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id true
+    # chunk_id_key chunk_id # Default value is "chunk_id".
+  </metadata>
+</match>
+```
+
+### include_chunk_id
+
+Whether including `chunk_id` for not. Default value is `false`.
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id true
+  </metadata>
+</match>
+```
+
+
+### chunk_id_key
+
+Specify `chunk_id_key` to store `chunk_id` information into records. Default value is `chunk_id`.
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id
+    chunk_id_key chunk_hex
+  </metadata>
+</match>
+```
+
 ## Configuration - Elasticsearch Input
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
@@ -1811,6 +1881,16 @@ template_name your-fluentd-template
 template_file /path/to/fluentd-template.json
 ```
 
+Note that if you create a new set of indexes every day, the elasticsearch ILM policy system will treat each day separately and will always
+maintain a separate active write index for each day.
+
+If you have a rollover based on max_age, it will continue to roll the indexes for prior dates even if no new documents are indexed.  If you want
+to delete indexes after a period of time, the ILM policy will never delete the current write index regardless of its age, so you would need a separate
+system, such as curator, to actually delete the old indexes.
+
+For this reason, if you put the date into the index names with ILM you should only rollover based on size or number of documents and may need to use
+curator to actually delete old indexes.
+
 #### Fixed ILM indices
 
 Also, users can use fixed ILM indices configuration.
@@ -1926,4 +2006,7 @@ Install dev dependencies:
 $ gem install bundler
 $ bundle install
 $ bundle exec rake test
+# To just run the test you are working on:
+$ bundle exec rake test TEST=test/plugin/test_out_elasticsearch.rb TESTOPTS='--verbose --name=test_custom_template_with_rollover_index_create_and_custom_ilm'
+
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.1.2'
+  s.version       = '4.2.2'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -22,7 +22,11 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_exists?(name, host = nil)
-    client(host).indices.get_template(:name => name)
+    if @use_legacy_template
+      client(host).indices.get_template(:name => name)
+    else
+      client(host).indices.get_index_template(:name => name)
+    end
     return true
   rescue Elasticsearch::Transport::Transport::Errors::NotFound
     return false
@@ -52,7 +56,11 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_put(name, template, host = nil)
-    client(host).indices.put_template(:name => name, :body => template)
+    if @use_legacy_template
+      client(host).indices.put_template(:name => name, :body => template)
+    else
+      client(host).indices.put_index_template(:name => name, :body => template)
+    end
   end
 
   def indexcreation(index_name, host = nil)
@@ -61,18 +69,19 @@ module Fluent::ElasticsearchIndexTemplate
     if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else
-      log.error("Error while index creation - #{index_name}: #{e.inspect}")
+      log.error("Error while index creation - #{index_name}", error: e)
     end
   end
 
-  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
     inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
     if overwrite
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
 
       log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
@@ -83,7 +92,8 @@ module Fluent::ElasticsearchIndexTemplate
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
       log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
     else
@@ -91,14 +101,15 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
     template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
     custom_template = if enable_ilm
                         inject_ilm_settings_to_template(deflector_alias_name,
                                                         target_index,
                                                         ilm_policy_id,
                                                         get_custom_template(template_file,
-                                                                            customize_template))
+                                                                            customize_template),
+                                                        index_separator)
                       else
                         get_custom_template(template_file, customize_template)
                       end
@@ -119,15 +130,31 @@ module Fluent::ElasticsearchIndexTemplate
     enable_ilm ? deflector_alias_name : template_name
   end
 
-  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
+  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
     log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
-    template.delete('template') if template.include?('template')
-    template['index_patterns'] = "#{target_index}-*"
-    template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
-    if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
-      log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+    template['index_patterns'] = "#{target_index}#{index_separator}*"
+    if @use_legacy_template
+      template.delete('template') if template.include?('template')
+      # Prepare settings Hash
+      if !template.key?('settings')
+        template['settings'] = {}
+      end
+      if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
+    else
+      # Prepare template.settings Hash
+      if !template['template'].key?('settings')
+        template['template']['settings'] = {}
+      end
+      if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
     end
-    template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
     template
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -8,6 +8,7 @@ rescue LoadError
 end
 require 'json'
 require 'uri'
+require 'base64'
 begin
   require 'strptime'
 rescue LoadError
@@ -17,6 +18,7 @@ require 'fluent/plugin/output'
 require 'fluent/event'
 require 'fluent/error'
 require 'fluent/time'
+require 'fluent/unique_id'
 require 'fluent/log-ext'
 require 'zlib'
 require_relative 'elasticsearch_constants'
@@ -57,6 +59,7 @@ module Fluent::Plugin
     attr_reader :template_names
     attr_reader :ssl_version_options
     attr_reader :compressable_connection
+    attr_reader :api_key_header
 
     helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
@@ -155,6 +158,7 @@ EOC
     config_param :default_elasticsearch_version, :integer, :default => DEFAULT_ELASTICSEARCH_VERSION
     config_param :log_es_400_reason, :bool, :default => false
     config_param :custom_headers, :hash, :default => {}
+    config_param :api_key, :string, :default => nil, :secret => true
     config_param :suppress_doc_wrap, :bool, :default => false
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
@@ -166,6 +170,12 @@ EOC
     config_param :ilm_policies, :hash, :default => {}
     config_param :ilm_policy_overwrite, :bool, :default => false
     config_param :truncate_caches_interval, :time, :default => nil
+    config_param :use_legacy_template, :bool, :default => true
+
+    config_section :metadata, param_name: :metainfo, multi: false do
+      config_param :include_chunk_id, :bool, :default => false
+      config_param :chunk_id_key, :string, :default => "chunk_id".freeze
+    end
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -212,6 +222,8 @@ EOC
         @remove_keys_on_update = @remove_keys_on_update.split ','
       end
 
+      @api_key_header = setup_api_key
+
       raise Fluent::ConfigError, "'max_retry_putting_template' must be greater than or equal to zero." if @max_retry_putting_template < 0
       raise Fluent::ConfigError, "'max_retry_get_es_version' must be greater than or equal to zero." if @max_retry_get_es_version < 0
 
@@ -247,7 +259,8 @@ EOC
             template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
           end
           verify_ilm_working if @enable_ilm
-        elsif @templates
+        end
+        if @templates
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             templates_hash_install(@templates, @template_overwrite)
           end
@@ -397,6 +410,12 @@ EOC
       end
     end
 
+    def setup_api_key
+      return {} unless @api_key
+
+      { "Authorization" => "ApiKey " + Base64.strict_encode64(@api_key) }
+    end
+
     def dry_run?
       if Fluent::Engine.respond_to?(:dry_run_mode)
         Fluent::Engine.dry_run_mode
@@ -557,7 +576,10 @@ EOC
                        else
                          {}
                        end
-        headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers).merge(gzip_headers)
+        headers = { 'Content-Type' => @content_type.to_s }
+                    .merge(@custom_headers)
+                    .merge(@api_key_header)
+                    .merge(gzip_headers)
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
 
         transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
@@ -755,6 +777,15 @@ EOC
       true
     end
 
+    def inject_chunk_id_to_record_if_needed(record, chunk_id)
+      if @metainfo&.include_chunk_id
+        record[@metainfo.chunk_id_key] = chunk_id
+        record
+      else
+        record
+      end
+    end
+
     def write(chunk)
       bulk_message_count = Hash.new { |h,k| h[k] = 0 }
       bulk_message = Hash.new { |h,k| h[k] = '' }
@@ -762,6 +793,7 @@ EOC
       meta = {}
 
       tag = chunk.metadata.tag
+      chunk_id = dump_unique_id_hex(chunk.unique_id)
       extracted_values = expand_placeholders(chunk)
       host = if @hosts
                extract_placeholders(@hosts, chunk)
@@ -771,6 +803,9 @@ EOC
 
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
+
+        record = inject_chunk_id_to_record_if_needed(record, chunk_id)
+
         begin
           meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
           info = if @include_index_in_url
@@ -947,22 +982,24 @@ EOC
 
     def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
       if template_name && @template_file
-        if !@logstash_format && @alias_indexes.include?(deflector_alias)
-          log.debug("Index alias #{deflector_alias} already exists (cached)")
-        elsif !@logstash_format && @template_names.include?(template_name)
-          log.debug("Template name #{template_name} already exists (cached)")
+        if !@logstash_format && (deflector_alias.nil? || (@alias_indexes.include? deflector_alias)) && (@template_names.include? template_name)
+          if deflector_alias
+            log.debug("Index alias #{deflector_alias} and template #{template_name} already exist (cached)")
+          else
+            log.debug("Template #{template_name} already exists (cached)")
+          end
         else
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             if customize_template
-              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
+              template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
             else
-              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index)
+              template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
             end
             ilm_policy = @ilm_policies[ilm_policy_id] || {}
             create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
           end
           @alias_indexes << deflector_alias unless deflector_alias.nil?
-          @template_names << template_name unless template_name.nil?
+          @template_names << template_name
         end
       end
     end