fluent-plugin-elasticsearch 4.0.8 → 4.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1979708479afe32ade449552ade837bc9cfc8acff343cd248008abc579b794d3
-  data.tar.gz: 6ed1a94f6e8553ca72fb75e319ee954f5a8a676ef4ccc0a6a65ca4d041c101c2
+  metadata.gz: 616365bcf32cc6d302718907d145639cbb4786b33cb8a837da24f3934cf7a837
+  data.tar.gz: 77df51b64aa07d12b57b82a093f4a5b8cbd1c1bf9c5c82384163118c050c68cf
 SHA512:
-  metadata.gz: e9957264a1ade9aa7e3b7cac60157c0adc9d9b2cd9e6d3d72ce73ca480e6ee32415bb94ad83c95ee7419e6b02b6695c28c39e2696945906e35e25c43421f70ca
-  data.tar.gz: 0efced0623b3312f1d767c82bb3ac9647e0082389bc6457052a4c74f5063a57ba21ff4effe881a18996c0a3f256ade017ec68b9df7423cb5c63869b3e49c1b07
+  metadata.gz: ae68cc2f0a2abbf7cf7a4c11a9bb5dac9a9e09ba9c053109748c671acd304198b45ad949782691a28eadb418e15bc66f1ef16897f810c5f0b5dd0ce04dd7913e
+  data.tar.gz: 484df0670840045d6c55bd1971142cf13a2a0c35cefc0ff96ada3259404d08529d97adbdd736bf9eaf99ad84c51d2124af1d0160c35042a736c875eef8aa5f3c

data/.github/workflows/issue-auto-closer.yml

@@ -0,0 +1,12 @@
+name: Autocloser
+on: [issues]
+jobs:
+  autoclose:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Autoclose issues that did not follow issue template
+      uses: roots/issue-closer-action@v1.1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
+        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)"

data/History.md

@@ -2,6 +2,29 @@
 
 ### [Unreleased]
 
+### 4.1.1
+- Correct ILM explain on logstash_format case (#786)
+
+### 4.1.0
+- Implement Fallback selector and configurable selector class (#782)
+
+### 4.0.11
+- Add custom and time placeholders combination testcase for ILM (#781)
+- Really support ILM on `logstash_format` enabled environment (#779)
+
+### 4.0.10
+- filter_elasticsearch_genid: Use entire record as hash seed (#777)
+- Suppress type in meta with suppress_type_name parameter (#774)
+- filter\_elasticsearch\_genid: Add hash generation mechanism from events (#773)
+- Clean up error text (#772)
+- Use GitHub Actions badges instead of Travis' (#760)
+- Add issue auto closer workflow (#759)
+- Document required permissions (#757)
+
+### 4.0.9
+- Add possibility to configure multiple ILM policies (#753)
+- Document required permissions (#757)
+
 ### 4.0.8
 - Handle compressable connection usable state (#743)
 - Use newer tls protocol versions (#739)

data/README.ElasticsearchGenID.md

@@ -0,0 +1,116 @@
+## Index
+
+* [Usage](#usage)
+* [Configuration](#configuration)
+  + [hash_id_key](#hash_id_key)
+  + [include_tag_in_seed](#include_tag_in_seed)
+  + [include_time_in_seed](#include_time_in_seed)
+  + [use_record_as_seed](#use_record_as_seed)
+  + [use_entire_record](#use_entire_record)
+  + [record_keys](#record_keys)
+  + [separator](#separator)
+  + [hash_type](#hash_type)
+* [Advanced Usage](#advanced-usage)
+
+## Usage
+
+In your Fluentd configuration, use `@type elasticsearch_genid`. Additional configuration is optional, default values would look like this:
+
+```
+<source>
+  @type elasticsearch_genid
+  hash_id_key _hash
+  include_tag_in_seed false
+  include_time_in_seed false
+  use_record_as_seed false
+  use_entire_record false
+  record_keys []
+  separator _
+  hash_type md5
+</match>
+```
+
+## Configuration
+
+### hash_id_key
+
+```
+hash_id_key _id
+```
+
+You can specify generated hash storing key.
+
+### include_tag_in_seed
+
+```
+include_tag_in_seed true
+```
+
+You can specify to use tag for hash generation seed.
+
+### include_time_in_seed
+
+```
+include_time_in_seed true
+```
+
+You can specify to use time for hash generation seed.
+
+### use_record_as_seed
+
+```
+use_record_as_seed true
+```
+
+You can specify to use record in events for hash generation seed. This parameter should be used with [record_keys](#record_keys) parameter in practice.
+
+### record_keys
+
+```
+record_keys request_id,pipeline_id
+```
+
+You can specify keys which are record in events for hash generation seed. This parameter should be used with [use_record_as_seed](#use_record_as_seed) parameter in practice.
+
+### use_entire_record
+
+```
+use_entire_record true
+```
+
+You can specify to use entire record in events for hash generation seed.
+
+
+### separator
+
+```
+separator |
+```
+
+You can specify separator charactor to creating seed for hash generation.
+
+### hash_type
+
+```
+hash_type sha1
+```
+
+You can specify hash algorithm.
+
+## Advanced Usage
+
+Elasticsearch GenID plugin can handle record contents differing with the following parameters:
+
+```aconf
+<filter the.awesome.your.routing.tag>
+  @type elasticsearch_genid
+  use_entire_record true
+  hash_type sha1
+  hash_id_key _hash
+  separator _
+  inc_time_as_key true
+  inc_tag_as_key true
+</filter>
+```
+
+The above configuration can handle tag, time, and record differing and generate different base64 encoded hash per record.

data/README.md

@@ -1,7 +1,9 @@
 # Fluent::Plugin::Elasticsearch, a plugin for [Fluentd](http://fluentd.org)
 
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elasticsearch.png)](http://badge.fury.io/rb/fluent-plugin-elasticsearch)
-[![Build Status](https://travis-ci.org/uken/fluent-plugin-elasticsearch.png?branch=master)](https://travis-ci.org/uken/fluent-plugin-elasticsearch)
+![Testing on Windows](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Windows/badge.svg?branch=master)
+![Testing on macOS](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20macOS/badge.svg?branch=master)
+![Testing on Ubuntu](https://github.com/uken/fluent-plugin-elasticsearch/workflows/Testing%20on%20Ubuntu/badge.svg?branch=master)
 [![Coverage Status](https://coveralls.io/repos/uken/fluent-plugin-elasticsearch/badge.png)](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
 [![Code Climate](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch.png)](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
 
@@ -31,6 +33,7 @@ Current maintainers: @cosmo0920
   + [time_key_exclude_timestamp](#time_key_exclude_timestamp)
   + [include_timestamp](#include_timestamp)
   + [utc_index](#utc_index)
+  + [suppress_type_name](#suppress_type_name)
   + [target_index_key](#target_index_key)
   + [target_type_key](#target_type_key)
   + [template_name](#template_name)
@@ -75,6 +78,7 @@ Current maintainers: @cosmo0920
   + [Hash flattening](#hash-flattening)
   + [Generate Hash ID](#generate-hash-id)
   + [sniffer_class_name](#sniffer-class-name)
+  + [selector_class_name](#selector-class-name)
   + [reload_after](#reload-after)
   + [validate_client_version](#validate-client-version)
   + [unrecoverable_error_types](#unrecoverable-error-types)
@@ -93,9 +97,12 @@ Current maintainers: @cosmo0920
   + [enable_ilm](#enable_ilm)
   + [ilm_policy_id](#ilm_policy_id)
   + [ilm_policy](#ilm_policy)
+  + [ilm_policies](#ilm_policies)
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
   + [truncate_caches_interval](#truncate_caches_interval)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
+* [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
+* [Elasticsearch permissions](#elasticsearch-permissions)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -349,6 +356,20 @@ utc_index true
 
 By default, the records inserted into index `logstash-YYMMDD` with UTC (Coordinated Universal Time). This option allows to use local time if you describe utc_index to false.
 
+### suppress_type_name
+
+In Elasticsearch 7.x, Elasticsearch cluster complains the following types removal warnings:
+
+```json
+{"type": "deprecation", "timestamp": "2020-07-03T08:02:20,830Z", "level": "WARN", "component": "o.e.d.a.b.BulkRequestParser", "cluster.name": "docker-cluster", "node.name": "70dd5c6b94c3", "message": "[types removal] Specifying types in bulk requests is deprecated.", "cluster.uuid": "NoJJmtzfTtSzSMv0peG8Wg", "node.id": "VQ-PteHmTVam2Pnbg7xWHw"  }
+```
+
+This can be suppressed with:
+
+```
+suppress_type_name true
+```
+
 ### target_index_key
 
 Tell this plugin to find the index name to write to in the record under this key in preference to other mechanisms. Key can be specified as path to nested record using dot ('.') as a separator.
@@ -458,6 +479,8 @@ deflector_alias test-current
 
 If [rollover_index](#rollover_index) is set, then this parameter will be in effect otherwise ignored.
 
+**NOTE:** Since 4.1.1, `deflector_alias` is prohibited to use with `enable_ilm`.
+
 ### index_prefix
 
 This parameter is marked as obsoleted.
@@ -969,7 +992,7 @@ reload_after 100
 
 #### Tips
 
-The included sniffer class does not required `out_elasticsearch`.
+The included sniffer class is not required `out_elasticsearch`.
 You should tell Fluentd where the sniffer class exists.
 
 If you use td-agent, you must put the following lines into `TD_AGENT_DEFAULT` file:
@@ -983,7 +1006,39 @@ If you use Fluentd directly, you must pass the following lines as Fluentd comman
 
 ```
 sniffer=$(td-agent-gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb)
-$ fluentd -r $sniffer" [AND YOUR OTHER OPTIONS]
+$ fluentd -r $sniffer [AND YOUR OTHER OPTIONS]
+```
+
+### Selector Class Name
+
+The default selector used by the `Elasticsearch::Transport` class works well when Fluentd should behave round robin and random selector cases. This doesn't work well when Fluentd should behave fallbacking from exhausted ES cluster to normal ES cluster.
+The parameter `selector_class_name` gives you the ability to provide your own Selector class to implement whatever selection nodes logic you require.
+
+The below configuration is using plugin built-in `ElasticseatchFallbackSelector`:
+
+```
+hosts exhausted-host:9201,normal-host:9200
+selector_class_name "Fluent::Plugin::ElasticseatchFallbackSelector"
+```
+
+#### Tips
+
+The included selector class is required in `out_elasticsearch` by default.
+But, your custom selector class is not required in `out_elasticsearch`.
+You should tell Fluentd where the selector class exists.
+
+If you use td-agent, you must put the following lines into `TD_AGENT_DEFAULT` file:
+
+```
+selector=/path/to/your_awesome_selector.rb
+TD_AGENT_OPTIONS="--use-v1-config -r $selector"
+```
+
+If you use Fluentd directly, you must pass the following lines as Fluentd command line option:
+
+```
+selector=/path/to/your_awesome_selector.rb
+$ fluentd -r $selector [AND YOUR OTHER OPTIONS]
 ```
 
 ### Reload After
@@ -1213,6 +1268,14 @@ Default value is `{}`.
 
 **NOTE:** This parameter requests to install elasticsearch-xpack gem.
 
+## ilm_policies
+
+A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
+
+Default value is `{}`.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
 ## ilm_policy_overwrite
 
 Specify whether overwriting ilm policy or not.
@@ -1233,6 +1296,36 @@ Default value is `nil`.
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
 
+## Configuration - Elasticsearch Filter GenID
+
+See [Elasticsearch Filter GenID document](README.ElasticsearchGenID.md)
+
+## Elasticsearch permissions
+
+If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
+
+The set of required permissions are the following:
+
+```json
+  "cluster": ["manage_index_templates", "monitor", "manage_ilm"],
+  "indices": [
+    {
+      "names": [ "*" ],
+      "privileges": ["write","create","delete","create_index","manage","manage_ilm"]
+    }
+  ]
+```
+
+These permissions can be narrowed down by:
+
+- Setting a more specific pattern for indices under the `names` field
+- Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
+- Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
+features in the plugin configuration
+
+The list of privileges along with their description can be found in
+[security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
+
 ## Troubleshooting
 
 ### Cannot send events to Elasticsearch
@@ -1662,7 +1755,7 @@ ILM target index alias is created with `index_name` or an index which is calcula
 
 From Elasticsearch plugin v4.0.0, ILM target index will be calculated from `index_name` (normal mode) or `logstash_prefix` (using with `logstash_format`as true).
 
-When using `deflector_alias` parameter, Elasticsearch plugin will create ILM target indices alias with `deflector_alias` instead of `index_name` or an index which is calculated from `logstash_prefix`. This behavior should be kept due to backward ILM feature compatibility.
+**NOTE:** Before Elasticsearch plugin v4.1.0, using `deflector_alias` parameter when ILM is enabled is permitted and handled, but, in the later releases such that 4.1.1 or later, it cannot use with when ILM is enabled.
 
 And also, ILM feature users should specify their Elasticsearch template for ILM enabled indices.
 Because ILM settings are injected into their Elasticsearch templates.
@@ -1675,7 +1768,13 @@ It usually should be used with default value which is `default`.
 
 Then, ILM parameters are used in alias index like as:
 
-`<index_name/logstash_prefix><index_separator><application_name>-000001`.
+##### Simple `index_name` case:
+
+`<index_name><index_separator><application_name>-000001`.
+
+##### `logstash_format` as `true` case:
+
+`<logstash_prefix><logstash_prefix_separator><application_name><logstash_prefix_separator><logstash_dateformat>-000001`.
 
 #### Example ILM settings
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.8'
+  s.version       = '4.1.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_fallback_selector.rb

@@ -0,0 +1,9 @@
+require 'elasticsearch/transport/transport/connections/selector'
+
+class Fluent::Plugin::ElasticseatchFallbackSelector
+  include Elasticsearch::Transport::Transport::Connections::Selector::Base
+
+  def select(options={})
+    connections.first
+  end
+end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -65,11 +65,12 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil)
+  def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
     inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
     if overwrite
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
+                                                                target_index,
                                                                 ilm_policy_id,
                                                                 get_template(template_file)) :
                      get_template(template_file), host)
@@ -80,6 +81,7 @@ module Fluent::ElasticsearchIndexTemplate
     if !template_exists?(inject_template_name, host)
       template_put(inject_template_name,
                    enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
+                                                                target_index,
                                                                 ilm_policy_id,
                                                                 get_template(template_file)) :
                      get_template(template_file), host)
@@ -89,10 +91,12 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host)
+  def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
     template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
     custom_template = if enable_ilm
-                        inject_ilm_settings_to_template(deflector_alias_name, ilm_policy_id,
+                        inject_ilm_settings_to_template(deflector_alias_name,
+                                                        target_index,
+                                                        ilm_policy_id,
                                                         get_custom_template(template_file,
                                                                             customize_template))
                       else
@@ -115,26 +119,30 @@ module Fluent::ElasticsearchIndexTemplate
     enable_ilm ? deflector_alias_name : template_name
   end
 
-  def inject_ilm_settings_to_template(deflector_alias_name, ilm_policy_id, template)
+  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
     log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
     template.delete('template') if template.include?('template')
-    template['index_patterns'] = "#{deflector_alias_name}-*"
-    template['order'] = template['order'] ? template['order'] + deflector_alias_name.split('-').length : 50 + deflector_alias_name.split('-').length
+    template['index_patterns'] = "#{target_index}-*"
+    template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
     if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
       log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
     end
-    template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias_name})
+    template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
     template
   end
 
-  def create_rollover_alias(index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite, host)
+  def create_rollover_alias(target_index, rollover_index, deflector_alias_name, app_name, index_date_pattern, index_separator, enable_ilm, ilm_policy_id, ilm_policy, ilm_policy_overwrite, host)
      # ILM request to create alias.
     if rollover_index || enable_ilm
       if !client.indices.exists_alias(:name => deflector_alias_name)
-        if index_date_pattern.empty?
-          index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-000001>'
+        if @logstash_format
+          index_name_temp = '<'+target_index+'-000001>'
         else
-          index_name_temp='<'+index_prefix.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
+          if index_date_pattern.empty?
+            index_name_temp = '<'+target_index.downcase+index_separator+app_name.downcase+'-000001>'
+          else
+            index_name_temp = '<'+target_index.downcase+index_separator+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
+          end
         end
         indexcreation(index_name_temp, host)
         body = rollover_alias_payload(deflector_alias_name)