fluent-plugin-elasticsearch 3.7.1 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66d34ebe18eabf21cd55dbe1c0f3d4f469298e3ada1072849068c1d490e62650
-  data.tar.gz: f91f6d38cc821642ddd6bc2fda232534b14d070fa9694db328bdef68d982d93a
+  metadata.gz: 28cfb80e32dd25c9d67c582d4d39672515bbc9e0acb6a7b940ee1e36edd35de1
+  data.tar.gz: 6c2440335c4a1ebc03939dee6204cbd9fc6265797c9658d728df70fc63e9ba9d
 SHA512:
-  metadata.gz: d83715d884fd22a0cb487e0eea16d37679800ce477bfcc888546d1856e48dd8bfa45106826731008b5b3d67cb2408a4d7eca9740a92076ec877a0f999252a512
-  data.tar.gz: d5e476b7ec45883dceeecffeb529d43c1b5a16a9a6c203139ed42891a8e702bc74861edcced7bfd87340e4f3150de0f5a7f7d9ecff8840d1dab9e1348db11601
+  metadata.gz: 7f67a2f892890ff80e34f6cd8aa14c5a61fc04095512f095d05554772eb384ddf54ec222006cf2b021ed08355d0af8575f0020671cd74de422c8b4c1e14989a0
+  data.tar.gz: 0d1919231ba8c8a88391a54d1853d992ba531f39c33826ff12ffd7258f9b3087ae5e18e1497edb0b6dac40998641a9ed79e28449cb90e2c9339a7a4e34406e4b

data/History.md

@@ -1,6 +1,11 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+### 3.8.0
+- Add FAQ for specifying index.codec (#679)
+- Add FAQ for connect_write timeout reached error (#687)
+- Unblocking buffer overflow with block action (#688)
+
 ### 3.7.1
 - Make conpatible for Fluentd v1.8 (#677)
 - Handle flatten_hashes in elasticsearch_dynamic (#675)

data/README.md

@@ -103,6 +103,8 @@ Current maintainers: @cosmo0920
   + [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
   + [Fluentd seems to hang if it unable to connect Elasticsearch, why?](#fluentd-seems-to-hang-if-it-unable-to-connect-elasticsearch-why)
   + [Enable Index Lifecycle Management](#enable-index-lifecycle-management)
+  + [How to specify index codec](#how-to-specify-index-codec)
+  + [Cannot push logs to Elasticsearch with connect_write timeout reached, why?](#cannot-push-logs-to-elasticsearch-with-connect_write-timeout-reached-why)
 * [Contact](#contact)
 * [Contributing](#contributing)
 * [Running tests](#running-tests)
@@ -1621,6 +1623,82 @@ customize_template {"<<index_prefix>>": "fluentd"}
 
 Note: This plugin only creates rollover-enabled indices, which are aliases pointing to them and index templates, and creates an ILM policy if enabled.
 
+### How to specify index codec
+
+Elasticsearch can handle compression methods for stored data such as LZ4 and best_compression.
+fluent-plugin-elasticsearch doesn't provide API which specifies compression method.
+
+Users can specify stored data compression method with template:
+
+Create `compression.json` as follows:
+
+```json
+{
+  "order": 100,
+  "index_patterns": [
+    "YOUR-INDEX-PATTERN"
+  ],
+  "settings": {
+    "index": {
+      "codec": "best_compression"
+    }
+  }
+}
+```
+
+Then, specify the above template in your configuration:
+
+```aconf
+template_name best_compression_tmpl
+template_file compression.json
+```
+
+Elasticsearch will store data with `best_compression`:
+
+```
+% curl -XGET 'http://localhost:9200/logstash-2019.12.06/_settings?pretty'
+```
+
+```json
+{
+  "logstash-2019.12.06" : {
+    "settings" : {
+      "index" : {
+        "codec" : "best_compression",
+        "number_of_shards" : "1",
+        "provided_name" : "logstash-2019.12.06",
+        "creation_date" : "1575622843800",
+        "number_of_replicas" : "1",
+        "uuid" : "THE_AWESOMEUUID",
+        "version" : {
+          "created" : "7040100"
+        }
+      }
+    }
+  }
+}
+```
+
+### Cannot push logs to Elasticsearch with connect_write timeout reached, why?
+
+It seems that Elasticsearch cluster is exhausted.
+
+Usually, Fluentd complains like the following log:
+
+```log
+2019-12-29 00:23:33 +0000 [warn]: buffer flush took longer time than slow_flush_log_threshold: elapsed_time=27.283766102716327 slow_flush_log_threshold=15.0 plugin_id="object:aaaffaaaaaff"
+2019-12-29 00:23:33 +0000 [warn]: buffer flush took longer time than slow_flush_log_threshold: elapsed_time=26.161768959928304 slow_flush_log_threshold=15.0 plugin_id="object:aaaffaaaaaff"
+2019-12-29 00:23:33 +0000 [warn]: buffer flush took longer time than slow_flush_log_threshold: elapsed_time=28.713624476008117 slow_flush_log_threshold=15.0 plugin_id="object:aaaffaaaaaff"
+2019-12-29 01:39:18 +0000 [warn]: Could not push logs to Elasticsearch, resetting connection and trying again. connect_write timeout reached
+2019-12-29 01:39:18 +0000 [warn]: Could not push logs to Elasticsearch, resetting connection and trying again. connect_write timeout reached
+```
+
+This warnings is usually caused by exhaused Elasticsearch cluster due to resource shortage.
+
+If CPU usage is spiked and Elasticsearch cluster is eating up CPU resource, this issue is caused by CPU resource shortage.
+
+Check your Elasticsearch cluster health status and resource usage.
+
 ## Contact
 
 If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.7.1'
+  s.version       = '3.8.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -33,6 +33,7 @@ module Fluent::Plugin
   class ElasticsearchOutput < Output
     class RecoverableRequestFailure < StandardError; end
     class UnrecoverableRequestFailure < Fluent::UnrecoverableError; end
+    class RetryStreamEmitFailure < StandardError; end
 
     # MissingIdFieldError is raised for records that do not
     # include the field for the unique record identifier
@@ -862,8 +863,15 @@ EOC
           error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
         end
       rescue RetryStreamError => e
+        log.trace "router.emit_stream for retry stream doing..."
         emit_tag = @retry_tag ? @retry_tag : tag
-        router.emit_stream(emit_tag, e.retry_stream)
+        # check capacity of buffer space
+        if retry_stream_retryable?
+          router.emit_stream(emit_tag, e.retry_stream)
+        else
+          raise RetryStreamEmitFailure, "buffer is full."
+        end
+        log.trace "router.emit_stream for retry stream done."
       rescue => e
         ignore = @ignore_exception_classes.any? { |clazz| e.class <= clazz }
 
@@ -879,6 +887,10 @@ EOC
       end
     end
 
+    def retry_stream_retryable?
+      @buffer.storable?
+    end
+
     def is_existing_connection(host)
       # check if the host provided match the current connection
       return false if @_es.nil?

data/test/plugin/test_out_elasticsearch.rb

@@ -3175,6 +3175,82 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal [['retry', 1, sample_record]], driver.events
   end
 
+  class FulfilledBufferRetryStreamTest < self
+    def test_bulk_error_retags_with_error_when_configured_and_fullfilled_buffer
+      def create_driver(conf='', es_version=5, client_version="\"5.0\"")
+        @client_version ||= client_version
+        Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+          def retry_stream_retryable?
+            false
+          end
+        CODE
+        # For request stub to detect compatibility.
+        @es_version ||= es_version
+        @client_version ||= client_version
+        if @es_version
+          Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+          def detect_es_major_version
+            #{@es_version}
+          end
+        CODE
+        end
+        Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+          def client_library_version
+            #{@client_version}
+          end
+        CODE
+        Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
+      end
+      driver = create_driver("retry_tag retry\n")
+      stub_request(:post, 'http://localhost:9200/_bulk')
+        .to_return(lambda do |req|
+                     { :status => 200,
+                       :headers => { 'Content-Type' => 'json' },
+                       :body => %({
+          "took" : 1,
+          "errors" : true,
+          "items" : [
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc1",
+                "status" : 403,
+                "error" : {
+                  "type" : "cluster_block_exception",
+                  "reason":"index [foo] blocked by: [FORBIDDEN/8/index write (api)]"
+                }
+              }
+            },
+            {
+              "create" : {
+                "_index" : "foo",
+                "_type"  : "bar",
+                "_id" : "abc2",
+                "status" : 403,
+                "error" : {
+                  "type" : "cluster_block_exception",
+                  "reason":"index [foo] blocked by: [FORBIDDEN/8/index write (api)]"
+                }
+              }
+            }
+           ]
+        })
+                     }
+                   end)
+
+      # Check buffer fulfillment condition
+      assert_raise(Fluent::Plugin::ElasticsearchOutput::RetryStreamEmitFailure) do
+        driver.run(default_tag: 'test') do
+          driver.feed(1, sample_record)
+          driver.feed(1, sample_record)
+        end
+      end
+
+      assert_equal [], driver.events
+    end
+  end
+
   def test_create_should_write_records_with_ids_and_skip_those_without
     driver.configure("write_operation create\nid_key my_id\n@log_level debug")
     stub_request(:post, 'http://localhost:9200/_bulk')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.7.1
+  version: 3.8.0
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-04 00:00:00.000000000 Z
+date: 2019-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd