fluent-plugin-elasticsearch 4.1.4 → 4.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7923225803a37c0076c7cb4c4d6e94c1ec9ccf120ce4067b45db8c0fc19f17f
-  data.tar.gz: f1ce4098fb1e4e563e00aae9fe8d30b2bc9cfcb08100616f0ff00b3dfa210667
+  metadata.gz: 1cd81dcd51b189a0197c440aee9211661cd277c668ecf84474540393d77e47f0
+  data.tar.gz: c8b22ada3eaa16f7423e04c804130f35a12c15de34565272a7048dda78fc0d83
 SHA512:
-  metadata.gz: '0579350a7eba583c7c4784f5500d9faa6df4e398b4b0d0e4aacf230502751a39fb9591d189217c869a82f6da689ffb435445aea7bd5920332787e787170b37d7'
-  data.tar.gz: 8d150716862d82b313a31b104842b368aac3f945bece0a11074bf43a25830c91e6b5ee0556a23e93fdc2159ab20185e08847b4c94bb10033fe7b259741b332b6
+  metadata.gz: 0564d3b40b180c425c68323dbaf7498e7483e4832a76262c8378ffdcb8bb4591e62b77a53a6a0c232ba418d397a2d74447d24b77441aca26a38aee02e0f389d2
+  data.tar.gz: f0274d59d1a6411ab320d93f3840325b1379f417f50c716ecba75ea0eda31392d75c1f41efcb958c825745ada6aaa3f6810da96aae4bfde7f11e6e03602242aa

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,37 @@
+---
+name:  Bug Report
+about: Create a report to help us improve. If you have questions about ES plugin on kubernetes, please direct these to https://discuss.kubernetes.io/ before sumbit kubernetes related issue.
+
+---
+
+(check apply)
+- [ ] read [the contribution guideline](https://github.com/uken/fluent-plugin-elasticsearch/blob/master/CONTRIBUTING.md)
+- [ ] (optional) already reported 3rd party upstream repository or mailing list if you use k8s addon or helm charts.
+
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Either clone and modify https://gist.github.com/pitr/9a518e840db58f435911
+
+**OR**
+
+Provide example config and message
+
+#### Expected Behavior or What you need to ask
+
+...
+
+#### Using Fluentd and ES plugin versions
+
+* OS version
+* Bare Metal or within Docker or Kubernetes or others?
+* Fluentd v0.12 or v0.14/v1.0
+  * paste result of ``fluentd --version`` or ``td-agent --version``
+* ES plugin 3.x.y/2.x.y or 1.x.y
+  * paste boot log of fluentd or td-agent
+  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
+* ES version (optional)
+* ES template(s) (optional)

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,24 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+(check apply)
+- [ ] read [the contribution guideline](https://github.com/uken/fluent-plugin-elasticsearch/blob/master/CONTRIBUTING.md)
+
+**Is your feature request related to a problem? Please describe.**
+
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+
+**Describe the solution you'd like**
+
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+
+<!-- Add any other context or screenshots about the feature request here. -->

data/.github/workflows/issue-auto-closer.yml

@@ -9,4 +9,4 @@ jobs:
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
-        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)"
+        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)|(.*Is your feature request related to a problem? Please describe.*)|(.*Describe the solution you'd like.*)|(.*Describe alternatives you've considered.*)"

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,6 +2,31 @@
 
 ### [Unreleased]
 
+### 4.3.1
+- Unsplit huge records by default (#851)
+
+### 4.3.0
+- Add cloud_id & cloud_auth settings (#850)
+- Fix failing tests on es6 client (#848)
+- feat: retry on transport errors (#846)
+- fix typo (#838)
+- Remove unnecessary nil check (#826)
+
+### 4.2.2
+- Remove unnecessary nil check (#826)
+- Fix ILM rollover index template pattern to apply index_separator (#825)
+- Fix ILM rollover alias creation when a placeholder is used in index_name (#823)
+- Add a note about the pitfalls of per-date indexes used with ILM (#822)
+
+### 4.2.1
+- Update a broken link (#821)
+- Include chunk_id in records (#820)
+- Fix a failing testcase (#818)
+
+### 4.2.0
+- ci: Add Ruby 2.7 jobs (#812)
+- Support Elasticsearch new style template (#810)
+
 ### 4.1.4
 - ElasticsearchGenID update docs for hash_type (#809)
 - Handle api key header (#808)

data/README.ElasticsearchGenID.md

@@ -17,7 +17,7 @@
 In your Fluentd configuration, use `@type elasticsearch_genid`. Additional configuration is optional, default values would look like this:
 
 ```
-<source>
+<filter>
   @type elasticsearch_genid
   hash_id_key _hash
   include_tag_in_seed false
@@ -27,7 +27,7 @@ In your Fluentd configuration, use `@type elasticsearch_genid`. Additional confi
   record_keys []
   separator _
   hash_type sha1
-</match>
+</filter>
 ```
 
 ## Configuration

data/README.md

@@ -19,6 +19,8 @@ Current maintainers: @cosmo0920
 * [Configuration](#configuration)
   + [host](#host)
   + [port](#port)
+  + [cloud_id](#cloud_id)
+  + [cloud_auth](#cloud_auth)
   + [emit_error_for_missing_id](#emit_error_for_missing_id)
   + [hosts](#hosts)
   + [user, password, path, scheme, ssl_verify](#user-password-path-scheme-ssl_verify)
@@ -101,6 +103,10 @@ Current maintainers: @cosmo0920
   + [ilm_policies](#ilm_policies)
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
   + [truncate_caches_interval](#truncate_caches_interval)
+  + [use_legacy_template](#use_legacy_template)
+  + [metadata section](#metadata-section)
+    + [include_chunk_id](#include_chunk_id)
+    + [chunk_id_key](#chunk_id_key)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
 * [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
 * [Elasticsearch permissions](#elasticsearch-permissions)
@@ -183,6 +189,26 @@ port 9201 # defaults to 9200
 
 You can specify Elasticsearch port by this parameter.
 
+### cloud_id
+
+```
+cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng== 
+```
+
+You can specify Elasticsearch cloud_id by this parameter.
+
+If you specify `cloud_id` option then `cloud_auth` option is required.
+If you specify `cloud_id` option, `host`, `port`, `user` and `password` options are ignored.
+
+### cloud_auth
+
+```
+cloud_auth 'elastic:slkjdaooewkd87iqQ2O8EQYV'
+```
+
+You can specify Elasticsearch cloud_auth by this parameter.
+
+
 ### emit_error_for_missing_id
 
 ```
@@ -497,7 +523,7 @@ Specify the application name for the rollover index to be created.
 application_name default # defaults to "default"
 ```
 
-If [enable_ilm](#enable_ilm is set, then this parameter will be in effect otherwise ignored.
+If [enable_ilm](#enable_ilm) is set, then this parameter will be in effect otherwise ignored.
 
 ### template_overwrite
 
@@ -920,7 +946,7 @@ Starting with version 0.8.0, this gem uses excon, which supports proxy with envi
 
 ### Buffer options
 
-`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/v0.14/articles/output-plugin-overview) and use `compat_parameters` plugin helper. It adds the following options:
+`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/output#overview) and use `compat_parameters` plugin helper. It adds the following options:
 
 ```
 buffer_type memory
@@ -1139,6 +1165,8 @@ And yet another option is described in Dynamic Configuration section.
 
 ### Dynamic configuration
 
+**NOTE**: *`out_elasticsearch_dynamic` will be planned to be marked as deprecated.* Please don't use the new Fluentd configuration. This plugin is maintained for backward compatibility.
+
 If you want configurations to depend on information in messages, you can use `elasticsearch_dynamic`. This is an experimental variation of the Elasticsearch plugin allows configuration values to be specified in ways such as the below:
 
 ```
@@ -1251,7 +1279,7 @@ Default value is `true`.
 
 Configure `bulk_message` request splitting threshold size.
 
-Default value is `20MB`. (20 * 1024 * 1024)
+Default value is `-1`(unlimited).
 
 If you specify this size as negative number, `bulk_message` request splitting feature will be disabled.
 
@@ -1303,6 +1331,63 @@ If it is set, timer for clearing `alias_indexes` and `template_names` caches wil
 
 Default value is `nil`.
 
+## use_legacy_template
+
+Use legacy template or not.
+
+Elasticsearch 7.8 or later supports the brand new composable templates.
+
+For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
+
+Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
+
+Please confirm that whether the using Elasticsearch cluster(s) support the composable template feature or not when turn on the brand new feature with this parameter.
+
+## <metadata\> section
+
+Users can specify whether including `chunk_id` information into records or not:
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id true
+    # chunk_id_key chunk_id # Default value is "chunk_id".
+  </metadata>
+</match>
+```
+
+### include_chunk_id
+
+Whether including `chunk_id` for not. Default value is `false`.
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id true
+  </metadata>
+</match>
+```
+
+
+### chunk_id_key
+
+Specify `chunk_id_key` to store `chunk_id` information into records. Default value is `chunk_id`.
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id
+    chunk_id_key chunk_hex
+  </metadata>
+</match>
+```
+
 ## Configuration - Elasticsearch Input
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
@@ -1820,6 +1905,16 @@ template_name your-fluentd-template
 template_file /path/to/fluentd-template.json
 ```
 
+Note that if you create a new set of indexes every day, the elasticsearch ILM policy system will treat each day separately and will always
+maintain a separate active write index for each day.
+
+If you have a rollover based on max_age, it will continue to roll the indexes for prior dates even if no new documents are indexed.  If you want
+to delete indexes after a period of time, the ILM policy will never delete the current write index regardless of its age, so you would need a separate
+system, such as curator, to actually delete the old indexes.
+
+For this reason, if you put the date into the index names with ILM you should only rollover based on size or number of documents and may need to use
+curator to actually delete old indexes.
+
 #### Fixed ILM indices
 
 Also, users can use fixed ILM indices configuration.
@@ -1935,4 +2030,7 @@ Install dev dependencies:
 $ gem install bundler
 $ bundle install
 $ bundle exec rake test
+# To just run the test you are working on:
+$ bundle exec rake test TEST=test/plugin/test_out_elasticsearch.rb TESTOPTS='--verbose --name=test_custom_template_with_rollover_index_create_and_custom_ilm'
+
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.1.4'
+  s.version       = '4.3.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -22,18 +22,23 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_exists?(name, host = nil)
-    client(host).indices.get_template(:name => name)
+    if @use_legacy_template
+      client(host).indices.get_template(:name => name)
+    else
+      client(host).indices.get_index_template(:name => name)
+    end
     return true
   rescue Elasticsearch::Transport::Transport::Errors::NotFound
     return false
   end
 
-  def retry_operate(max_retries, fail_on_retry_exceed = true)
+  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
+    transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
-    rescue *client.transport.host_unreachable_exceptions, Timeout::Error => e
+    rescue *client.transport.host_unreachable_exceptions, *transport_errors, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries
@@ -52,7 +57,11 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_put(name, template, host = nil)
-    client(host).indices.put_template(:name => name, :body => template)
+    if @use_legacy_template
+      client(host).indices.put_template(:name => name, :body => template)
+    else
+      client(host).indices.put_index_template(:name => name, :body => template)
+    end
   end
 
   def indexcreation(index_name, host = nil)
@@ -65,14 +74,15 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
     inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
     if overwrite
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
 
       log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
@@ -83,7 +93,8 @@ module Fluent::ElasticsearchIndexTemplate
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
                                                                 target_index,
                                                                 ilm_policy_id,
-                                                                get_template(template_file)) :
+                                                                get_template(template_file),
+                                                                index_separator) :
                      get_template(template_file), host)
       log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
     else
@@ -91,14 +102,15 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
     template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
     custom_template = if enable_ilm
                         inject_ilm_settings_to_template(deflector_alias_name,
                                                         target_index,
                                                         ilm_policy_id,
                                                         get_custom_template(template_file,
-                                                                            customize_template))
+                                                                            customize_template),
+                                                        index_separator)
                       else
                         get_custom_template(template_file, customize_template)
                       end
@@ -119,15 +131,31 @@ module Fluent::ElasticsearchIndexTemplate
     enable_ilm ? deflector_alias_name : template_name
   end
 
-  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
+  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
     log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
-    template.delete('template') if template.include?('template')
-    template['index_patterns'] = "#{target_index}-*"
-    template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
-    if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
-      log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+    template['index_patterns'] = "#{target_index}#{index_separator}*"
+    if @use_legacy_template
+      template.delete('template') if template.include?('template')
+      # Prepare settings Hash
+      if !template.key?('settings')
+        template['settings'] = {}
+      end
+      if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
+    else
+      # Prepare template.settings Hash
+      if !template['template'].key?('settings')
+        template['template']['settings'] = {}
+      end
+      if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
     end
-    template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
     template
   end