fluent-plugin-elasticsearch 4.0.4 → 4.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4253b332821c59318a743dccc063cfe0bf5b6f5302727d0bf974280b41336ea
-  data.tar.gz: 8731c0ba69bb200b1c397fc87ad0237c16ca8f5a825d37d16c0844664afd6292
+  metadata.gz: f76e6d69d85cace43250e57e272d26e492e4ae5d48856306687efc487279dd4d
+  data.tar.gz: c6f583de5857aba3c9a01bdde72e16c851d52cdc07b1151052155356954155ec
 SHA512:
-  metadata.gz: 41eff72fc9b79075b3e3c2812c43fc34d4e789ca005bbe0d722e6d75ef9d01706841bba7214495a1d1bcdc3eeb5d86008c002e4aa4686508b22f1caafce731a3
-  data.tar.gz: 3087f4fa5c6c008092366652ee388690bd012c6528ab01eb699fe352760a584500c83ea903d22b40693bf3d141bf7ed220f491cd64be3be08fa9a7ca2308f246
+  metadata.gz: 58003e22128dc44d615351ca0b36bd6bc2495d21b146f08713db6f8cfc77323192c38c841df446f30ec6d0edde89cced9c51a676f523e44d7488444ec9f2e737
+  data.tar.gz: 9b34567bea973a765fb2dbf699dadd26d80fd50b45b7b1179469d60425c67602e3d288f8da25305c089c20eb6daf236b900931ca0abd503d2665a5d6ab153c12

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: Testing on Ubuntu
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/macos.yml

@@ -0,0 +1,26 @@
+name: Testing on macOS
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6' ]
+        os:
+          - macOS-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/windows.yml

@@ -0,0 +1,26 @@
+name: Testing on Windows
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6' ]
+        os:
+          - windows-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.travis.yml

@@ -1,17 +1,39 @@
 language: ruby
 
-matrix:
+jobs:
   include:
     - rvm: 2.4.6
       gemfile: Gemfile
+      os: linux
+      arch: amd64
     - rvm: 2.5.5
       gemfile: Gemfile
+      os: linux
+      arch: amd64
     - rvm: 2.6.3
-      gemfile: gemfiles/Gemfile.ilm
+      gemfile: gemfiles/Gemfile.without.ilm
+      os: linux
+      arch: amd64
+    - rvm: 2.6.3
+      gemfile: gemfiles/Gemfile.elasticsearch.v6
+      os: linux
+      arch: amd64
+    - rvm: 2.6.3
+      gemfile: Gemfile
+      os: linux
+      arch: amd64
+    - rvm: 2.6.3
+      gemfile: Gemfile
+      os: linux
+      arch: arm64
     - rvm: 2.6.3
       gemfile: Gemfile
+      os: osx
+      osx_image: xcode11.3
     - rvm: 2.7.0
       gemfile: Gemfile
+      os: linux
+      arch: amd64
 
 gemfile:
  - Gemfile
@@ -20,4 +42,3 @@ before_install:
   - gem update --system=2.7.8
 
 script: bundle exec rake test
-sudo: false

data/Gemfile

@@ -4,6 +4,8 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'simplecov', require: false
-gem 'coveralls', require: false
+gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
+gem "elasticsearch-xpack"
+gem "oj"

data/History.md

@@ -2,6 +2,31 @@
 
 ### [Unreleased]
 
+### 4.0.9
+- Add possibility to configure multiple ILM policies (#753)
+- Document required permissions (#757)
+
+### 4.0.8
+- Handle compressable connection usable state (#743)
+- Use newer tls protocol versions (#739)
+- Add GitHub Actions file (#740)
+
+### 4.0.7
+- Added http_backend_excon_nonblock config in out_elasticsearch (#733)
+
+### 4.0.6
+- Add fallback mechanism for handling to detect es version (#730)
+- Remove needless section (#728)
+- Handle exception if index already exists (#727)
+- Tweak test cases (#726)
+
+### 4.0.5
+-  add logstash_dateformat as placeholder (#718)
+- Tweak travis.yml for suppressing validator warnings and add CI for Linux Arm64 architecture and macOS 10.14 (#724)
+- Elasticsearch ruby v7.5 (#723)
+- Add Oj serializer testcases for all job (#722)
+- Update documentation for ILM (#721)
+
 ### 4.0.4
 - Provide clearing caches timer (#719)
 

data/README.md

@@ -45,6 +45,7 @@ Current maintainers: @cosmo0920
   + [templates](#templates)
   + [max_retry_putting_template](#max_retry_putting_template)
   + [fail_on_putting_template_retry_exceed](#fail_on_putting_template_retry_exceed)
+  + [fail_on_detecting_es_version_retry_exceed](#fail_on_detecting_es_version_retry_exceed)
   + [max_retry_get_es_version](#max_retry_get_es_version)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
@@ -65,6 +66,7 @@ Current maintainers: @cosmo0920
   + [content_type](#content_type)
   + [include_index_in_url](#include_index_in_url)
   + [http_backend](#http_backend)
+  + [http_backend_excon_nonblock](#http_backend_excon_nonblock)
   + [prefer_oj_serializer](#prefer_oj_serializer)
   + [compression_level](#compression_level)
   + [Client/host certificate options](#clienthost-certificate-options)
@@ -91,15 +93,16 @@ Current maintainers: @cosmo0920
   + [enable_ilm](#enable_ilm)
   + [ilm_policy_id](#ilm_policy_id)
   + [ilm_policy](#ilm_policy)
+  + [ilm_policies](#ilm_policies)
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
   + [truncate_caches_interval](#truncate_caches_interval)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
+* [Elasticsearch permissions](#elasticsearch-permissions)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
   + [Cannot connect TLS enabled reverse Proxy](#cannot-connect-tls-enabled-reverse-proxy)
   + [Declined logs are resubmitted forever, why?](#declined-logs-are-resubmitted-forever-why)
-  + [Suggested to increase flush_thread_count, why?](#suggested-to-increase-flush_thread_count-why)
   + [Suggested to install typhoeus gem, why?](#suggested-to-install-typhoeus-gem-why)
   + [Stopped to send events on k8s, why?](#stopped-to-send-events-on-k8s-why)
   + [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
@@ -502,6 +505,23 @@ If you have multiple output plugin, you could use this property to do not fail o
 fail_on_putting_template_retry_exceed false # defaults to true
 ```
 
+### fail_on_detecting_es_version_retry_exceed
+
+Indicates whether to fail when `max_retry_get_es_version` is exceeded.
+If you want to use fallback mechanism for obtaining ELasticsearch version, you could use this property to do not fail on fluentd statup.
+
+```
+fail_on_detecting_es_version_retry_exceed false
+```
+
+And the following parameters should be working with:
+
+```
+verify_es_version_at_startup true
+max_retry_get_es_version 2 # greater than 0.
+default_elasticsearch_version 7 # This version is used when occurring fallback.
+```
+
 ### max_retry_get_es_version
 
 You can specify times of retry obtaining Elasticsearch version.
@@ -799,6 +819,21 @@ Default value is `excon` which is default http_backend of elasticsearch plugin.
 http_backend typhoeus
 ```
 
+### http_backend_excon_nonblock
+
+With `http_backend_excon_nonblock false`, elasticsearch plugin use excon with nonblock=false.
+If you use elasticsearch plugin with jRuby for https, you may need to consider to set `false` to avoid follwoing problems.
+- https://github.com/geemus/excon/issues/106
+- https://github.com/jruby/jruby-ossl/issues/19
+
+But for all other case, it strongly reccomend to set `true` to avoid process hangin problem reported in https://github.com/uken/fluent-plugin-elasticsearch/issues/732
+
+Default value is `true`.
+
+```
+http_backend_excon_nonblock false
+```
+
 ### compression_level
 You can add gzip compression of output data. In this case `default_compression`, `best_compression` or `best speed` option should be chosen.
 By default there is no compression, default value for this option is `no_compression`
@@ -848,7 +883,11 @@ ssl_min_version TLSv1_2
 
 Elasticsearch plugin will use TLSv1.2 as minimum ssl version and TLSv1.3 as maximum ssl version on transportation with TLS. Note that when they are used in Elastissearch plugin configuration, *`ssl_version` is not used* to set up TLS version.
 
-If they are *not* specified in the Elasticsearch plugin configuration, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
+If they are *not* specified in the Elasticsearch plugin configuration, `ssl_max_version` and `ssl_min_version` is set up with:
+
+In Elasticsearch plugin v4.0.8 or later with Ruby 2.5 or later environment, `ssl_max_version` should be `TLSv1_3` and `ssl_min_version` should be `TLSv1_2`.
+
+From Elasticsearch plugin v4.0.4 to v4.0.7 with Ruby 2.5 or later environment, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
 
 
 ### Proxy Support
@@ -1176,6 +1215,14 @@ Default value is `{}`.
 
 **NOTE:** This parameter requests to install elasticsearch-xpack gem.
 
+## ilm_policies
+
+A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
+
+Default value is `{}`.
+
+**NOTE:** This parameter requests to install elasticsearch-xpack gem.
+
 ## ilm_policy_overwrite
 
 Specify whether overwriting ilm policy or not.
@@ -1196,6 +1243,32 @@ Default value is `nil`.
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
 
+## Elasticsearch permissions
+
+If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
+
+The set of required permissions are the following:
+
+```json
+  "cluster": ["manage_index_templates", "monitor", "manage_ilm"],
+  "indices": [
+    {
+      "names": [ "*" ],
+      "privileges": ["write","create","delete","create_index","manage","manage_ilm"]
+    }
+  ]
+```
+
+These permissions can be narrowed down by:
+
+- Setting a more specific pattern for indices under the `names` field
+- Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
+- Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
+features in the plugin configuration
+
+The list of privileges along with their description can be found in
+[security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
+
 ## Troubleshooting
 
 ### Cannot send events to Elasticsearch
@@ -1445,30 +1518,6 @@ The following configuration uses label:
 </label>
 ```
 
-### Suggested to increase flush_thread_count, why?
-
-fluent-plugin-elasticsearch default behavior has a possibility to cause events traffic jam.
-When users use `flush_thread_count` = 1, ES plugin retries to send events if connection errors are disappeared.
-
-To prevent the following warning and sending events blocking, you must specify `flush_thread_count` >= 2:
-
-```log
-2018-12-24 14:32:06 +0900 [warn]: #0 To prevent events traffic jam, you should specify 2 or more 'flush_thread_count'.
-```
-
-```aconf
-<match out.elasticsearch.**>
-  @type elasticsearch
-  host localhost
-  port 9200
-  # ...
-  <buffer tag>
-    @type memory # or file
-    flush_thread_count 4
-  </buffer>
-</match>
-```
-
 ### Suggested to install typhoeus gem, why?
 
 fluent-plugin-elasticsearch doesn't depend on typhoeus gem by default.
@@ -1639,17 +1688,31 @@ Main ILM feature parameters are:
 * `ilm_policy_id`
 * `ilm_policy`
 
+* Advanced usage parameters
+  * `application_name`
+  * `index_separator`
+
 They are not all mandatory parameters but they are used for ILM feature in effect.
 
 ILM target index alias is created with `index_name` or an index which is calculated from `logstash_prefix`.
 
 From Elasticsearch plugin v4.0.0, ILM target index will be calculated from `index_name` (normal mode) or `logstash_prefix` (using with `logstash_format`as true).
 
-When using `deflectoe_alias` parameter, Elasticsearch plugin will create ILM target indices alias with `deflector_alias` instead of `index_name` or an index which is calculated from `logstash_prefix`. This behavior should be kept due to backward ILM feature compatibility.
+When using `deflector_alias` parameter, Elasticsearch plugin will create ILM target indices alias with `deflector_alias` instead of `index_name` or an index which is calculated from `logstash_prefix`. This behavior should be kept due to backward ILM feature compatibility.
 
 And also, ILM feature users should specify their Elasticsearch template for ILM enabled indices.
 Because ILM settings are injected into their Elasticsearch templates.
 
+`application_name` and `index_separator` also affect alias index names.
+
+But this parameter is prepared for advanced usage.
+
+It usually should be used with default value which is `default`.
+
+Then, ILM parameters are used in alias index like as:
+
+`<index_name/logstash_prefix><index_separator><application_name>-000001`.
+
 #### Example ILM settings
 
 ```aconf

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.4'
+  s.version       = '4.0.9'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/gemfiles/Gemfile.elasticsearch.v6

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
+gemspec :path => "../"
+
+gem 'simplecov', require: false
+gem 'coveralls', ">= 0.8.0", require: false
+gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
+gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
+gem "elasticsearch", "~> 6.8.1"
+gem "elasticsearch-xpack"
+gem "oj"

data/gemfiles/{Gemfile.ilm → Gemfile.without.ilm}

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 gemspec :path => "../"
 
 gem 'simplecov', require: false
-gem 'coveralls', require: false
+gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
-gem "elasticsearch-xpack"
+gem "oj"

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -58,7 +58,7 @@ module Fluent::ElasticsearchIndexTemplate
   def indexcreation(index_name, host = nil)
     client(host).indices.create(:index => index_name)
   rescue Elasticsearch::Transport::Transport::Error => e
-    if e.message =~ /"already exists"/
+    if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else
       log.error("Error while index creation - #{index_name}: #{e.inspect}")

data/lib/fluent/plugin/elasticsearch_tls.rb

@@ -10,7 +10,7 @@ module Fluent::Plugin
                                [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
                              end
 
-    DEFAULT_VERSION = :TLSv1
+    DEFAULT_VERSION = :TLSv1_2
     METHODS_MAP = begin
                     # When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
                     map = {
@@ -48,8 +48,8 @@ module Fluent::Plugin
       if USE_TLS_MINMAX_VERSION
         case
         when ssl_min_version.nil? && ssl_max_version.nil?
-          ssl_min_version = METHODS_MAP[ssl_version]
-          ssl_max_version = METHODS_MAP[ssl_version]
+          ssl_min_version = METHODS_MAP[:TLSv1_2]
+          ssl_max_version = METHODS_MAP[:TLSv1_3]
         when ssl_min_version && ssl_max_version.nil?
           raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
         when ssl_min_version.nil? && ssl_max_version