fluent-plugin-elasticsearch 2.12.2 → 2.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d97d7d622a14ab5bf55b08dfb20792ce7893988b9f47abc013d33afbb735166
-  data.tar.gz: 4e585b0c87f929d083e9c20c137f22da6f5260d24032bfd9baf4c8fc8c109287
+  metadata.gz: dbe9d45b32f6a3db242d3f41dfa7c9c56ee425c9f36bbea1866c8fe61795f214
+  data.tar.gz: 7070955261a6a34a2d27d97fd7c2b040d03d8ec725d7d58fedcda863a4aaa083
 SHA512:
-  metadata.gz: 1a5c1b5b7b27575c92350f9338a39f684d5f7facfb3f4ad182642c9a210e5dea912d575e8a35b763c14f7d019ecf3cb9982bc95434436b9fe378a2fdf30704cb
-  data.tar.gz: 573c3b6c23ddbe42173952f914f3e702c3a8eb9ff839e78e108bb372b786351bd47383d05fa7628a065de8a10718fba1e4c5220d624ae4cdf831abd106ff4031
+  metadata.gz: 220580fb2c1405e355f49a30249ec1c5cb3f3774789d440057f3377f6493e26ca46569cd3081c7dc16493ea8056eb7fd12919d0c11137439e8bff55eb0749611
+  data.tar.gz: c90e537194d8986180fa183c757a19694c638dc324e9edc9b8fb46a355991f06efc08acfac7af47bb3587a3c66fb3b1a969361ae64cf5c9fa589d6aa00f6c7bd

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 2.12.3
+- Added log_es_400_reason configuration item (#511)
+- Allow a user to specify the rollover index date pattern (#510)
+
 ### 2.12.2
 - Verify connection at startup (#504)
 - Add faq for glob pattern tag routing (#502)

data/README.md

@@ -36,6 +36,7 @@ Current maintainers: @cosmo0920
   + [template_overwrite](#template_overwrite)
   + [customize_template](#customize_template)
   + [rollover_index](#rollover_index)
+  + [index_date_pattern](#index_date_pattern)
   + [deflector_alias](#deflector_alias)
   + [application_name](#application_name)
   + [index_prefix](#index_prefix)
@@ -76,6 +77,7 @@ Current maintainers: @cosmo0920
   + [Dynamic configuration](#dynamic-configuration)
   + [Placeholders](#placeholders)
   + [Multi workers](#multi-workers)
+  + [log_es_400_reason](#log-es-400-reason)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -367,6 +369,19 @@ rollover_index true # defaults to false
 
 If [customize_template](#customize_template) is set, then this parameter will be in effect otherwise ignored.
 
+### index_date_pattern
+
+Specify this to override the index date pattern for creating a rollover index. The default is to use "now/d",
+for example: <logstash-default-{now/d}-000001>. Overriding this changes the rollover time period. Setting
+"now/w{xxxx.ww}" would create weekly rollover indexes instead of daily.
+
+This setting only takes effect when combined with the [rollover_index](#rollover_index) setting.
+```
+index_date_pattern "now/w{xxxx.ww}" # defaults to "now/d"
+```
+
+If [customize_template](#customize_template) is set, then this parameter will be in effect otherwise ignored.
+
 ### deflector_alias
 
 Specify the deflector alias which would be assigned to the rollover index created. This is useful in case of using the Elasticsearch rollover API 
@@ -970,6 +985,12 @@ Since Fluentd v0.14, multi workers feature had been implemented to increase thro
 </system>
 ```
 
+## log_es_400_reason
+
+By default, the error logger won't record the reason for a 400 error from the Elasticsearch API unless you set log_level to debug. However, this results in a lot of log spam, which isn't desirable if all you want is the 400 error reasons. You can set this `true` to capture the 400 error reasons without all the other debug logs.
+
+Default value is `false`.
+
 ## Troubleshooting
 
 ### Cannot send events to Elasticsearch

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.12.2'
+  s.version       = '2.12.3'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -23,6 +23,14 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     unrecoverable_error_types.include?(type)
   end
 
+  def log_es_400_reason(&block)
+    if @plugin.log_es_400_reason
+      block.call
+    else
+      @plugin.log.on_debug &block
+    end
+  end
+
   def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
     items = response['items']
     if items.nil? || !items.is_a?(Array)
@@ -74,7 +82,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
       when 400 == status
         stats[:bad_argument] += 1
         reason = ""
-        @plugin.log.on_debug do
+        log_es_400_reason do
           if item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
             reason = " [error type]: #{item[write_operation]['error']['type']}"
           end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -68,7 +68,7 @@ module Fluent::ElasticsearchIndexTemplate
     end
   end
 
-  def template_custom_install(template_name, template_file, overwrite, customize_template, index_prefix, rollover_index, deflector_alias_name, app_name)
+  def template_custom_install(template_name, template_file, overwrite, customize_template, index_prefix, rollover_index, deflector_alias_name, app_name, index_date_pattern)
     template_custom_name=template_name.downcase
     if overwrite
       template_put(template_custom_name, get_custom_template(template_file, customize_template))
@@ -83,7 +83,7 @@ module Fluent::ElasticsearchIndexTemplate
     end
     if rollover_index
       if !client.indices.exists_alias(:name => deflector_alias_name)
-        index_name_temp='<'+index_prefix.downcase+'-'+app_name.downcase+'-{now/d}-000001>'
+        index_name_temp='<'+index_prefix.downcase+'-'+app_name.downcase+'-{'+index_date_pattern+'}-000001>'
         indexcreation(index_name_temp)
         client.indices.put_alias(:index => index_name_temp, :name => deflector_alias_name)
         log.info("The alias '#{deflector_alias_name}' is created for the index '#{index_name_temp}'")

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -100,6 +100,7 @@ EOC
     config_param :template_overwrite, :bool, :default => false
     config_param :customize_template, :hash, :default => nil
     config_param :rollover_index, :string, :default => false
+    config_param :index_date_pattern, :string, :default => "now/d"
     config_param :deflector_alias, :string, :default => nil
     config_param :index_prefix, :string, :default => "logstash"
     config_param :application_name, :string, :default => "default"
@@ -126,6 +127,7 @@ EOC
     config_param :unrecoverable_error_types, :array, :default => ["out_of_memory_error", "es_rejected_execution_exception"]
     config_param :verify_es_version_at_startup, :bool, :default => true
     config_param :default_elasticsearch_version, :integer, :default => DEFAULT_ELASTICSEARCH_VERSION
+    config_param :log_es_400_reason, :bool, :default => false
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -166,14 +168,14 @@ EOC
       end
 
       raise Fluent::ConfigError, "'max_retry_putting_template' must be positive number." if @max_retry_putting_template < 0
-      
+
       if @template_name && @template_file
         retry_install(@max_retry_putting_template) do
           if @customize_template
             if @rollover_index
               raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
             end
-            template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @index_prefix, @rollover_index, @deflector_alias, @application_name)
+            template_custom_install(@template_name, @template_file, @template_overwrite, @customize_template, @index_prefix, @rollover_index, @deflector_alias, @application_name, @index_date_pattern)
           else
             template_install(@template_name, @template_file, @template_overwrite)
           end

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -9,11 +9,13 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     attr_reader :log
     attr_reader :write_operation, :error_events
     attr_accessor :unrecoverable_error_types
-    def initialize(log)
+    attr_accessor :log_es_400_reason
+    def initialize(log, log_es_400_reason = false)
       @log = log
       @write_operation = 'index'
       @error_events = []
       @unrecoverable_error_types = ["out_of_memory_error", "es_rejected_execution_exception"]
+      @log_es_400_reason = log_es_400_reason
     end
 
     def router
@@ -101,6 +103,48 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
   end
 
+  class TEST400ResponseReasonNoDebug < self
+    def setup
+      Fluent::Test.setup
+      @log_device = Fluent::Test::DummyLogDevice.new
+      dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
+      logger = ServerEngine::DaemonLogger.new(@log_device, dl_opts)
+      @log = Fluent::Log.new(logger)
+      @plugin = TestPlugin.new(@log)
+      @handler = Fluent::Plugin::ElasticsearchErrorHandler.new(@plugin)
+      @plugin.log_es_400_reason = true
+    end
+
+    def test_400_responses_reason_log
+      records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
+      response = parse_response(%({
+      "took" : 0,
+      "errors" : true,
+      "items" : [
+        {
+          "create" : {
+            "_index" : "foo",
+            "status" : 400,
+            "error" : {
+              "type"  : "mapper_parsing_exception",
+              "reason" : "failed to parse"
+            }
+          }
+        }
+      ]
+     }))
+      chunk = MockChunk.new(records)
+      dummy_extracted_values = []
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      assert_equal(1, @plugin.error_events.size)
+      expected_log = "failed to parse"
+      exception_message = @plugin.error_events.first[:error].message
+      assert_true(exception_message.include?(expected_log),
+                  "Exception do not contain '#{exception_message}' '#{expected_log}'")
+      assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
+    end
+  end
+
   def test_dlq_400_responses
     records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
     response = parse_response(%({

data/test/plugin/test_out_elasticsearch.rb

@@ -220,6 +220,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal ["out_of_memory_error", "es_rejected_execution_exception"], instance.unrecoverable_error_types
     assert_true instance.verify_es_version_at_startup
     assert_equal Fluent::Plugin::ElasticsearchOutput::DEFAULT_ELASTICSEARCH_VERSION, instance.default_elasticsearch_version
+    assert_false instance.log_es_400_reason
   end
 
   test 'configure Content-Type' do
@@ -418,6 +419,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       template_file   #{template_file}
       customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
       rollover_index  true
+      index_date_pattern now/w{xxxx.ww}
       deflector_alias myapp_deflector
       index_prefix    mylogs
       application_name myapp
@@ -433,13 +435,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/myapp_alias_template").
       to_return(:status => 200, :body => "", :headers => {})
     # creation of index which can rollover
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E").
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
       to_return(:status => 200, :body => "", :headers => {})
     # check if alias exists
     stub_request(:head, "https://john:doe@logs.google.com:777/es//_alias/myapp_deflector").
       to_return(:status => 404, :body => "", :headers => {})
     # put the alias for the index
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fd%7D-000001%3E/_alias/myapp_deflector").
+    stub_request(:put, "https://john:doe@logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
       to_return(:status => 200, :body => "", :headers => {})
     
     driver(config)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.12.2
+  version: 2.12.3
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-24 00:00:00.000000000 Z
+date: 2018-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd