fluent-plugin-elasticsearch 1.18.1 → 1.18.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4e4ff948915da51a4c267d0350157b841c6b3f09862571d9777072d60d5c337
-  data.tar.gz: dde99b5bd72bfadad60013689d8109b5776beb6ae942d65fad1ad8777df1ecb0
+  metadata.gz: ef82d990d532395ac70cfaca633811ecfec6e9277f02544a2fc73402aea79bd9
+  data.tar.gz: 30beccac0549c496631ba4943c2aae0161db08bb55966f38c1554b8c493beb12
 SHA512:
-  metadata.gz: 13d604ed0ac1822333b5464efe34fd7f943b9c1151cbd2bf60c40356b028bab9164c6c66d26dd515c182d19a3f95f4b18e504d118212bf86be647b3681bd5659
-  data.tar.gz: d90733254d79a977a12adc64a47f01315acfe391c3841f82c055be6aef20b1c209dd6e0d86620a581efd1b8b5f90b433fb6fb640126cda122b427802fc1beec3
+  metadata.gz: a7aaed653bb448e943265718258e9f92e2cb17fb778684483e58bac51c7a4175821aec96da2b6956ccf0f1b2e5c81775672f0c9ad34120e86b6e1e5f7b71aff2
+  data.tar.gz: 37a07ba5b227ce441396984abbedbafd6005d709df2b1489b39fcd5cdbb4e4aee820104b0cf99c944990133a227daa2ccde2b0a0d5f27a07929aa112e980d069

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 1.18.2
+- Retry upsert on recoverable error. (porting to v0.12) #667 (#684)
+
 ### 1.18.1
 - add new option to suppress doc wrapping (porting to v0.12) (#558)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.18.1'
+  s.version       = '1.18.2'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/log-ext.rb

@@ -0,0 +1,38 @@
+require 'fluent/log'
+# For elasticsearch-ruby v7.0.0 or later
+# logger for Elasticsearch::Loggable required the following methods:
+#
+# * debug?
+# * info?
+# * warn?
+# * error?
+# * fatal?
+
+module Fluent
+  class Log
+    # Elasticsearch::Loggable does not request trace? method.
+    # def trace?
+    #   @level <= LEVEL_TRACE
+    # end
+
+    def debug?
+      @level <= LEVEL_DEBUG
+    end
+
+    def info?
+      @level <= LEVEL_INFO
+    end
+
+    def warn?
+      @level <= LEVEL_WARN
+    end
+
+    def error?
+      @level <= LEVEL_ERROR
+    end
+
+    def fatal?
+      @level <= LEVEL_FATAL
+    end
+  end
+end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -40,6 +40,8 @@ class Fluent::ElasticsearchErrorHandler
         write_operation = @plugin.write_operation
       elsif INDEX_OP == @plugin.write_operation && item.has_key?(CREATE_OP)
         write_operation = CREATE_OP
+      elsif UPSERT_OP == @plugin.write_operation && item.has_key?(UPDATE_OP)
+        write_operation = UPDATE_OP
       else
         # When we don't have an expected ops field, something changed in the API
         # expected return values (ES 2.x)

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -11,6 +11,7 @@ end
 
 require 'fluent/output'
 require 'fluent/event'
+require 'fluent/log-ext'
 require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -7,7 +7,8 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   class TestPlugin
     attr_reader :log
-    attr_reader :write_operation, :error_events
+    attr_reader :error_events
+    attr_accessor :write_operation
     def initialize(log)
       @log = log
       @write_operation = 'index'
@@ -80,21 +81,13 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
 
-  def test_retry_error
-    records = []
-    error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
-    10.times do |i|
-      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
-    end
-    chunk = MockChunk.new(records)
-
-    response = parse_response(%({
+  def create_mock_response(method)
+    parse_response(%({
       "took" : 1,
       "errors" : true,
       "items" : [
         {
-          "create" : {
+          "#{method}" : {
             "_index" : "foo",
             "_type"  : "bar",
             "_id" : "1",
@@ -102,7 +95,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           }
         },
         {
-          "create" : {
+          "#{method}" : {
             "_index" : "foo",
             "_type"  : "bar",
             "_id" : "2",
@@ -114,15 +107,19 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           }
         },
         {
-          "create" : {
+          "#{method}" : {
             "_index" : "foo",
             "_type"  : "bar",
             "_id" : "3",
-            "status" : 409
+            "status" : 409,
+            "error" : {
+              "type":"version_conflict_engine_exception",
+              "reason":"document already exists"
+            }
           }
         },
         {
-          "create" : {
+          "#{method}" : {
             "_index" : "foo",
             "_type"  : "bar",
             "_id" : "5",
@@ -133,7 +130,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           }
         },
         {
-          "create" : {
+          "#{method}" : {
             "_index" : "foo",
             "_type"  : "bar",
             "_id" : "6",
@@ -145,7 +142,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           }
         },
         {
-          "create" : {
+          "#{method}" : {
             "_index" : "foo",
             "_type"  : "bar",
             "_id" : "7",
@@ -157,7 +154,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
           }
         },
         {
-          "create" : {
+          "#{method}" : {
             "_index" : "foo",
             "_type"  : "bar",
             "_id" : "8",
@@ -170,7 +167,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
         }
       ]
     }))
+  end
+
+  def test_retry_error_index
+    records = []
+    error_records = Hash.new(false)
+    error_records.merge!({0=>true, 4=>true, 9=>true})
+    10.times do |i|
+      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
+    end
+    chunk = MockChunk.new(records)
 
+    response = create_mock_response('create')
     begin
       failed = false
       @handler.handle_error(response, 'atag', chunk, response['items'].length)
@@ -195,6 +203,42 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
   end
 
+  def test_retry_error_upsert
+    @plugin.write_operation = 'upsert'
+    records = []
+    error_records = Hash.new(false)
+    error_records.merge!({0=>true, 4=>true, 9=>true})
+    10.times do |i|
+      records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
+    end
+    chunk = MockChunk.new(records)
+
+    response = create_mock_response('update')
+    begin
+      failed = false
+      @handler.handle_error(response, 'atag', chunk, response['items'].length)
+    rescue Fluent::ElasticsearchOutput::RetryStreamError=>e
+      failed = true
+      records = [].tap do |records|
+        e.retry_stream.each {|time, record| records << record}
+      end
+      assert_equal 4, records.length
+      assert_equal 2, records[0]['_id']
+      # upsert is retried in case of conflict error.
+      assert_equal 3, records[1]['_id']
+      assert_equal 6, records[2]['_id']
+      assert_equal 8, records[3]['_id']
+      error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
+      assert_equal 2, error_ids.length
+      assert_equal 5, error_ids[0]
+      assert_equal 7, error_ids[1]
+      @plugin.error_events.collect {|h| h[:error]}.each do |e|
+        assert_true e.respond_to?(:backtrace)
+      end
+    end
+    assert_true failed
+  end
+
   def test_old_es_1_X_responses
     records = [{time: 123, record: {"foo" => "bar", '_id' => 'abc'}}]
     response = parse_response(%({

data/test/test_log-ext.rb

@@ -0,0 +1,33 @@
+require 'helper'
+require 'fluent/log-ext'
+
+class TestFluentLogExtHandler < Test::Unit::TestCase
+  def setup
+    @log = Fluent::Test::TestLogger.new
+    @log.level = "info"
+  end
+
+  def test_trace?
+    assert_false @log.respond_to?(:trace?)
+  end
+
+  def test_debug?
+    assert_true @log.respond_to?(:debug?)
+  end
+
+  def test_info?
+    assert_true @log.respond_to?(:info?)
+  end
+
+  def test_warn?
+    assert_true @log.respond_to?(:warn?)
+  end
+
+  def test_error?
+    assert_true @log.respond_to?(:error?)
+  end
+
+  def test_fatal?
+    assert_true @log.respond_to?(:fatal?)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.18.1
+  version: 1.18.2
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-08 00:00:00.000000000 Z
+date: 2019-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -144,6 +144,7 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-elasticsearch.gemspec
+- lib/fluent/log-ext.rb
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
@@ -157,6 +158,7 @@ files:
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json
+- test/test_log-ext.rb
 homepage: https://github.com/uken/fluent-plugin-elasticsearch
 licenses:
 - Apache-2.0
@@ -176,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector
@@ -187,3 +189,4 @@ test_files:
 - test/plugin/test_out_elasticsearch.rb
 - test/plugin/test_out_elasticsearch_dynamic.rb
 - test/plugin/test_template.json
+- test/test_log-ext.rb