fluent-plugin-elasticsearch 1.9.0.rc.1 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac0ae4b2396f3c809f7cb7e59fe8dbe1b26bf516
-  data.tar.gz: 6354f35f3a2ef593c5d1e4e3bf5213f6d0d17efa
+  metadata.gz: 9b756e1f1c69c5e12941b6d96242c64a09553e65
+  data.tar.gz: b9153e7eaf0b4b3532dd89eb9f7c2cb5f58533a0
 SHA512:
-  metadata.gz: 55af560dafa19e24f62819c95944673113ec8b9541f0bf338938fadfabf8b9cad48850c201f829365761739278fdde2356887eef1ebe13ae62f267b7946efc6b
-  data.tar.gz: 71cd49d06e9e664d0143117669b8aab67efa228271b952bc2cae296ab69011fbe749553611f306a2cf44d7e7001ca5b08da7d23ddd8bc7892225d292f6f0553e
+  metadata.gz: 942deb8934cecb911d1f1751760ac1484bc303bbb03079953c66e0eff9a24644c7a0df1517af5436ceca4bc0afed068b4d8a502250492be400e1e412332ddc47
+  data.tar.gz: a4cc8d9c20624d9773859cf1546a446e575f4486ec8f80d27dbfa323108c183e2c39cf75650c31cf96fb4c447469227d936682b2311c07090e5f5a500d191670

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 1.9.0
+- add `time_parse_error_tag` (#211)
+- add `reconnect_on_error` (#214)
+
 ### 1.9.0.rc.1
 - Optimize output plugins (#203)
 

data/README.md

@@ -41,6 +41,8 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [remove_keys_on_update](#remove_keys_on_update)
   + [remove_keys_on_update_key](#remove_keys_on_update_key)
   + [write_operation](#write_operation)
+  + [time_parse_error_tag](#time_parse_error_tag)
+  + [reconnect_on_error](#reconnect_on_error)
   + [Client/host certificate options](#clienthost-certificate-options)
   + [Proxy Support](#proxy-support)
   + [Buffered output options](#buffered-output-options)
@@ -392,6 +394,21 @@ The write_operation can be any of:
 
 **Please note, id is required in create, update, and upsert scenario. Without id, the message will be dropped.**
 
+### time_parse_error_tag
+
+With `logstash_format true`, elasticsearch plugin parses timestamp field for generating index name. If the record has invalid timestamp value, this plugin emits an error event to `@ERROR` label with `time_parse_error_tag` configured tag.
+
+Default value is `Fluent::ElasticsearchOutput::TimeParser.error` for backward compatibility. `::` separated tag is not good for tag routing because some plugins assume tag is separated by `.`. We recommend to set this parameter like `time_parse_error_tag es_plugin.output.time.error`.
+We will change default value to `.` separated tag.
+
+### reconnect_on_error
+Indicates that the plugin should reset connection on any error (reconnect on next send).
+By default it will reconnect only on "host unreachable exceptions".
+We recommended to set this true in the presence of elasticsearch shield.
+```
+reconnect_on_error true # defaults to false
+```
+
 ### Client/host certificate options
 
 Need to verify ElasticSearch's certificate?  You can use the following parameter to specify a CA instead of using an environment variable.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.9.0.rc.1'
+  s.version       = '1.9.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{ElasticSearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -58,17 +58,18 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :templates, :hash, :default => nil
   config_param :include_tag_key, :bool, :default => false
   config_param :tag_key, :string, :default => 'tag'
+  config_param :time_parse_error_tag, :string, :default => 'Fluent::ElasticsearchOutput::TimeParser.error'
+  config_param :reconnect_on_error, :bool, :default => false
 
   include Fluent::ElasticsearchIndexTemplate
 
   def initialize
     super
-    @time_parser = TimeParser.new(@time_key_format, @router)
   end
 
   def configure(conf)
     super
-    @time_parser = TimeParser.new(@time_key_format, @router)
+    @time_parser = create_time_parser
 
     if @remove_keys
       @remove_keys = @remove_keys.split(/\s*,\s*/)
@@ -110,40 +111,32 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     result
   end
 
-  def start
-    super
-  end
-
   # once fluent v0.14 is released we might be able to use
   # Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives
   # [sec,nsec] where as we want something we can call `strftime` on...
-  class TimeParser
-    def initialize(time_key_format, router)
-      @time_key_format = time_key_format
-      @router = router
-      @parser = if time_key_format
-        begin
-          # Strptime doesn't support all formats, but for those it does it's
-          # blazingly fast.
-          strptime = Strptime.new(time_key_format)
-          Proc.new { |value| strptime.exec(value).to_datetime }
-        rescue
-          # Can happen if Strptime doesn't recognize the format; or
-          # if strptime couldn't be required (because it's not installed -- it's
-          # ruby 2 only)
-          Proc.new { |value| DateTime.strptime(value, time_key_format) }
-        end
-      else
-        Proc.new { |value| DateTime.parse(value) }
+  def create_time_parser
+    if @time_key_format
+      begin
+        # Strptime doesn't support all formats, but for those it does it's
+        # blazingly fast.
+        strptime = Strptime.new(@time_key_format)
+        Proc.new { |value| strptime.exec(value).to_datetime }
+      rescue
+        # Can happen if Strptime doesn't recognize the format; or
+        # if strptime couldn't be required (because it's not installed -- it's
+        # ruby 2 only)
+        Proc.new { |value| DateTime.strptime(value, @time_key_format) }
       end
+    else
+      Proc.new { |value| DateTime.parse(value) }
     end
+  end
 
-    def parse(value, event_time)
-      @parser.call(value)
-    rescue => e
-      @router.emit_error_event("Fluent::ElasticsearchOutput::TimeParser.error", Fluent::Engine.now, {'time' => event_time, 'format' => @time_key_format, 'value' => value }, e)
-      return Time.at(event_time).to_datetime
-    end
+  def parse_time(value, event_time, tag)
+    @time_parser.call(value)
+  rescue => e
+    router.emit_error_event(@time_parse_error_tag, Fluent::Engine.now, {'tag' => tag, 'time' => event_time, 'format' => @time_key_format, 'value' => value}, e)
+    return Time.at(event_time).to_datetime
   end
 
   def client
@@ -215,10 +208,6 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     end.join(', ')
   end
 
-  def shutdown
-    super
-  end
-
   BODY_DELIMITER = "\n".freeze
   UPDATE_OP = "update".freeze
   UPSERT_OP = "upsert".freeze
@@ -302,11 +291,12 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
         target_index = target_index_parent.delete(target_index_child_key)
       elsif @logstash_format
         if record.has_key?(TIMESTAMP_FIELD)
-          dt = record[TIMESTAMP_FIELD]
-          dt = @time_parser.parse(record[TIMESTAMP_FIELD], time)
+          rts = record[TIMESTAMP_FIELD]
+          dt = parse_time(rts, time, tag)
         elsif record.has_key?(@time_key)
-          dt = @time_parser.parse(record[@time_key], time)
-          record[TIMESTAMP_FIELD] = record[@time_key] unless time_key_exclude_timestamp
+          rts = record[@time_key]
+          dt = parse_time(rts, time, tag)
+          record[TIMESTAMP_FIELD] = rts unless @time_key_exclude_timestamp
         else
           dt = Time.at(time).to_datetime
           record[TIMESTAMP_FIELD] = dt.to_s
@@ -370,6 +360,9 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
         retry
       end
       raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+    rescue Exception
+      @_es = nil if @reconnect_on_error
+      raise
     end
   end
 end

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -16,6 +16,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
   config_param :reload_on_failure, :string, :default => "false"
   config_param :resurrect_after, :string, :default => "60"
   config_param :ssl_verify, :string, :default => "true"
+  config_param :reconnect_on_error, :bool, :default => false
 
   def configure(conf)
     super
@@ -207,6 +208,9 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
         retry
       end
       raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+    rescue Exception
+      @_es = nil if @reconnect_on_error
+      raise
     end
   end
 

data/test/plugin/test_out_elasticsearch.rb

@@ -367,8 +367,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_writes_to_target_index_key_logstash
-    driver.configure("target_index_key @target_index\n")
-    driver.configure("logstash_format true\n")
+    driver.configure("target_index_key @target_index
+                      logstash_format true")
     time = Time.parse Date.today.to_s
     stub_elastic_ping
     stub_elastic
@@ -378,8 +378,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
    def test_writes_to_target_index_key_logstash_uppercase
-    driver.configure("target_index_key @target_index\n")
-    driver.configure("logstash_format true\n")
+    driver.configure("target_index_key @target_index
+                      logstash_format true")
     time = Time.parse Date.today.to_s
     stub_elastic_ping
     stub_elastic
@@ -400,8 +400,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_writes_to_target_index_key_fallack_logstash
-    driver.configure("target_index_key @target_index\n")
-    driver.configure("logstash_format true\n")
+    driver.configure("target_index_key @target_index\n
+                      logstash_format true")
     time = Time.parse Date.today.to_s
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
@@ -441,8 +441,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_writes_to_target_type_key_fallack_to_type_name
-    driver.configure("target_type_key @target_type\n")
-    driver.configure("type_name mytype\n")
+    driver.configure("target_type_key @target_type
+                      type_name mytype")
     stub_elastic_ping
     stub_elastic
     driver.emit(sample_record)
@@ -724,9 +724,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
-  def test_uses_custom_time_key_format_logs_an_error
+  data(:default => nil,
+       :custom_tag => 'es_plugin.output.time.error')
+  def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
+    tag_config = tag_for_error ? "time_parse_error_tag #{tag_for_error}" : ''
+    tag_for_error = 'Fluent::ElasticsearchOutput::TimeParser.error' if tag_for_error.nil?
     driver.configure("logstash_format true
-                      time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
+                      time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n#{tag_config}\n")
     stub_elastic_ping
     stub_elastic
 
@@ -737,7 +741,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run
 
     log = driver.instance.router.emit_error_handler.log
-    errors = log.out.logs.grep /tag="Fluent::ElasticsearchOutput::TimeParser.error"/
+    errors = log.out.logs.grep /tag="#{tag_for_error}"/
     assert_equal(1, errors.length, "Error was logged for timestamp parse failure")
 
     assert_equal(index, index_cmds[0]['index']['_index'])
@@ -910,6 +914,54 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(connection_resets, 3)
   end
 
+  def test_reconnect_on_error_enabled
+    connection_resets = 0
+
+    stub_elastic_ping(url="http://localhost:9200").with do |req|
+      connection_resets += 1
+    end
+
+    stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
+    end
+    
+    driver.configure("reconnect_on_error true\n")
+    driver.emit(sample_record)
+
+    assert_raise(ZeroDivisionError) {
+      driver.run
+    }
+
+    assert_raise(ZeroDivisionError) {
+      driver.run
+    }
+    assert_equal(connection_resets, 2)
+  end
+
+  def test_reconnect_on_error_disabled
+    connection_resets = 0
+
+    stub_elastic_ping(url="http://localhost:9200").with do |req|
+      connection_resets += 1
+    end
+
+    stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
+    end
+    
+    driver.configure("reconnect_on_error false\n")
+    driver.emit(sample_record)
+
+    assert_raise(ZeroDivisionError) {
+      driver.run
+    }
+
+    assert_raise(ZeroDivisionError) {
+      driver.run
+    }
+    assert_equal(connection_resets, 1)
+  end
+
   def test_update_should_not_write_if_theres_no_id
     driver.configure("write_operation update\n")
     stub_elastic_ping

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -538,6 +538,54 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal(connection_resets, 3)
   end
 
+  def test_reconnect_on_error_enabled
+    connection_resets = 0
+
+    stub_elastic_ping(url="http://localhost:9200").with do |req|
+      connection_resets += 1
+    end
+
+    stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
+    end
+    
+    driver.configure("reconnect_on_error true\n")
+    driver.emit(sample_record)
+
+    assert_raise(ZeroDivisionError) {
+      driver.run
+    }
+
+    assert_raise(ZeroDivisionError) {
+      driver.run
+    }
+    assert_equal(connection_resets, 2)
+  end
+
+  def test_reconnect_on_error_disabled
+    connection_resets = 0
+
+    stub_elastic_ping(url="http://localhost:9200").with do |req|
+      connection_resets += 1
+    end
+
+    stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
+    end
+    
+    driver.configure("reconnect_on_error false\n")
+    driver.emit(sample_record)
+
+    assert_raise(ZeroDivisionError) {
+      driver.run
+    }
+
+    assert_raise(ZeroDivisionError) {
+      driver.run
+    }
+    assert_equal(connection_resets, 1)
+  end
+
   def test_update_should_not_write_if_theres_no_id
     driver.configure("write_operation update\n")
     stub_elastic_ping

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.9.0.rc.1
+  version: 1.9.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-03 00:00:00.000000000 Z
+date: 2016-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -151,9 +151,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1