fluent-plugin-elasticsearch 2.12.1 → 2.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d653e88c7896e738bd345f3f5732c1f41f251b9372bf62fb0da758769b891b5
-  data.tar.gz: 574a4d8d57c0f98a667f853efa0214c6e952b8b30b30197f000037ca7f3044c4
+  metadata.gz: 6d97d7d622a14ab5bf55b08dfb20792ce7893988b9f47abc013d33afbb735166
+  data.tar.gz: 4e585b0c87f929d083e9c20c137f22da6f5260d24032bfd9baf4c8fc8c109287
 SHA512:
-  metadata.gz: 249b1fe3782ec8ce4401c671b24c85d239608c214574b3b692d62e7f201c68d4b4f85d28779e6259c25a5e05b212961eb8ef7c900d541b2c28ceaae3a62d5ead
-  data.tar.gz: 2d1841c7f6d6f71a63daabb78f3b75266be7543e6f6cf03e686beebcaf7aee4a9678f6885aface2fc1c12c6d695ea2185c6f689faef5e01c981793a5ee433cd3
+  metadata.gz: 1a5c1b5b7b27575c92350f9338a39f684d5f7facfb3f4ad182642c9a210e5dea912d575e8a35b763c14f7d019ecf3cb9982bc95434436b9fe378a2fdf30704cb
+  data.tar.gz: 573c3b6c23ddbe42173952f914f3e702c3a8eb9ff839e78e108bb372b786351bd47383d05fa7628a065de8a10718fba1e4c5220d624ae4cdf831abd106ff4031

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 2.12.2
+- Verify connection at startup (#504)
+- Add faq for glob pattern tag routing (#502)
+
 ### 2.12.1
 - Make configurable unrecoverable types (#501)
 - Add FAQ for TLS enabled nginx proxy TLS version incompatibility trouble (#496)

data/ISSUE_TEMPLATE.md

@@ -17,10 +17,10 @@ Provide example config and message
 #### Using Fluentd and ES plugin versions
 
 * OS version
+* Bare Metal or within Docker or Kubernetes or others?
 * Fluentd v0.12 or v0.14/v1.0
   * paste result of ``fluentd --version`` or ``td-agent --version``
 * ES plugin 2.x.y or 1.x.y
   * paste boot log of fluentd or td-agent
   * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
 * ES version (optional)
-* Bear Metal or Within Docker or Kubernetes or others? (optional)

data/README.md

@@ -70,6 +70,8 @@ Current maintainers: @cosmo0920
   + [reload_after](#reload_after)
   + [validate_client_version](#validate-client-version)
   + [unrecoverable_error_types](#unrecoverable-error-types)
+  + [verify_es version at startup](#verify_es_version_at_startup)
+  + [default_elasticsearch_version](#default_elasticsearch_version)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
   + [Placeholders](#placeholders)
@@ -78,6 +80,7 @@ Current maintainers: @cosmo0920
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
   + [Cannot connect TLS enabled reverse Proxy](#cannot-connect-tls-enabled-reverse-proxy)
+  + [Declined logs are resubmitted forever, why?](#declined-logs-are-resubmitted-forever-why)
 * [Contact](#contact)
 * [Contributing](#contributing)
 * [Running tests](#running-tests)
@@ -844,6 +847,27 @@ Then, remove `es_rejected_execution_exception` from `unrecoverable_error_types`
 unrecoverable_error_types ["out_of_memory_error"]
 ```
 
+### verify_es_version_at_startup
+
+Because Elasticsearch plugin should change behavior each of Elasticsearch major versions.
+
+For example, Elasticsearch 6 starts to prohibit multiple type_names in one index, and Elasticsearch 7 will handle only `_doc` type_name in index.
+
+If you want to disable to verify Elasticsearch version at start up, set it as `false`.
+
+When using the following configuration, ES plugin intends to communicate into Elasticsearch 6.
+
+```
+verify_es_version_at_startup false
+default_elasticsearch_version 6
+```
+
+The default value is `true`.
+
+### default_elasticsearch_version
+
+This parameter changes that ES plugin assumes default Elasticsearch version. The default value is `5`.
+
 ### Not seeing a config you need?
 
 We try to keep the scope of this plugin small and not add too many configuration options. If you think an option would be useful to others, feel free to open an issue or contribute a Pull Request.
@@ -1093,6 +1117,94 @@ If you want to use TLS v1.2, please use `ssl_version` parameter like as:
 ssl_version TLSv1_2
 ```
 
+### Declined logs are resubmitted forever, why?
+
+Sometimes users write Fluentd configuration like this:
+
+```aconf
+<match **>
+  @type elasticsearch
+  host localhost
+  port 9200
+  type_name fluentd
+  logstash_format true
+  time_key @timestamp
+  include_timestamp true
+  reconnect_on_error true
+  reload_on_failure true
+  reload_connections false
+  request_timeout 120s
+</match>
+```
+
+The above configuration does not use [`@label` feature](https://docs.fluentd.org/v1.0/articles/config-file#(5)-group-filter-and-output:-the-%E2%80%9Clabel%E2%80%9D-directive) and use glob(**) pattern.
+It is usually problematic configuration.
+
+In error scenario, error events will be emitted with `@ERROR` label, and `fluent.*` tag.
+The black hole glob pattern resubmits a problematic event into pushing Elasticsearch pipeline.
+
+This situation causes flood of declined log:
+
+```log
+2018-11-13 11:16:27 +0000 [warn]: #0 dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error="400 - Rejected by Elasticsearch" location=nil tag="app.fluentcat" time=2018-11-13 11:16:17.492985640 +0000 record={"message"=>"\xFF\xAD"}
+2018-11-13 11:16:38 +0000 [warn]: #0 dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error="400 - Rejected by Elasticsearch" location=nil tag="fluent.warn" time=2018-11-13 11:16:27.978851140 +0000 record={"error"=>"#<Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError: 400 - Rejected by Elasticsearch>", "location"=>nil, "tag"=>"app.fluentcat", "time"=>2018-11-13 11:16:17.492985640 +0000, "record"=>{"message"=>"\xFF\xAD"}, "message"=>"dump an error event: error_class=Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchError error=\"400 - Rejected by Elasticsearch\" location=nil tag=\"app.fluentcat\" time=2018-11-13 11:16:17.492985640 +0000 record={\"message\"=>\"\\xFF\\xAD\"}"}
+```
+
+Then, user should use more concrete tag route or use `@label`.
+The following sections show two examples how to solve flood of declined log.
+One is using concrete tag routing, the other is using label routing.
+
+#### Using concrete tag routing
+
+The following configuration uses concrete tag route:
+
+```aconf
+<match out.elasticsearch.**>
+  @type elasticsearch
+  host localhost
+  port 9200
+  type_name fluentd
+  logstash_format true
+  time_key @timestamp
+  include_timestamp true
+  reconnect_on_error true
+  reload_on_failure true
+  reload_connections false
+  request_timeout 120s
+</match>
+```
+
+#### Using label feature
+
+The following configuration uses label:
+
+```aconf
+<source>
+  @type forward
+  @label @ES
+</source>
+<label @ES>
+  <match out.elasticsearch.**>
+    @type elasticsearch
+    host localhost
+    port 9200
+    type_name fluentd
+    logstash_format true
+    time_key @timestamp
+    include_timestamp true
+    reconnect_on_error true
+    reload_on_failure true
+    reload_connections false
+    request_timeout 120s
+  </match>
+</label>
+<label @ERROR>
+  <match **>
+    @type stdout
+  </match>
+</label>
+```
+
 ## Contact
 
 If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.12.1'
+  s.version       = '2.12.2'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -124,6 +124,8 @@ EOC
     config_param :validate_client_version, :bool, :default => false
     config_param :prefer_oj_serializer, :bool, :default => false
     config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "es_rejected_execution_exception"]
+    config_param :verify_es_version_at_startup, :bool, :default => true
+    config_param :default_elasticsearch_version, :integer, :default => DEFAULT_ELASTICSEARCH_VERSION
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -216,11 +218,15 @@ EOC
       end
 
       @last_seen_major_version =
-        begin
-          detect_es_major_version
-        rescue
-          log.warn "Could not connect Elasticsearch or obtain version. Assuming Elasticsearch 5."
-          DEFAULT_ELASTICSEARCH_VERSION
+        if @verify_es_version_at_startup
+          begin
+            detect_es_major_version
+          rescue
+            log.warn "Could not connect Elasticsearch or obtain version. Assuming Elasticsearch #{@default_elasticsearch_version}."
+            @default_elasticsearch_version
+          end
+        else
+          @default_elasticsearch_version
         end
       if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
         log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
@@ -500,7 +506,7 @@ EOC
 
       tag = chunk.metadata.tag
       extracted_values = expand_placeholders(chunk.metadata)
-      @last_seen_major_version = detect_es_major_version rescue DEFAULT_ELASTICSEARCH_VERSION
+      @last_seen_major_version = detect_es_major_version rescue @default_elasticsearch_version
 
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash

data/test/plugin/test_out_elasticsearch.rb

@@ -217,6 +217,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal "fluentd", default_type_name
     assert_equal :excon, instance.http_backend
     assert_false instance.prefer_oj_serializer
+    assert_equal ["out_of_memory_error", "es_rejected_execution_exception"], instance.unrecoverable_error_types
+    assert_true instance.verify_es_version_at_startup
+    assert_equal Fluent::Plugin::ElasticsearchOutput::DEFAULT_ELASTICSEARCH_VERSION, instance.default_elasticsearch_version
   end
 
   test 'configure Content-Type' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.12.1
+  version: 2.12.2
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-19 00:00:00.000000000 Z
+date: 2018-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd