fluent-plugin-elasticsearch 5.0.3 → 5.1.1

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -27,13 +27,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
        @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       return [meta, header, record]
     end
 
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      indices
+    end
+
     def append_record_to_messages(op, meta, header, record, msgs)
       if record.has_key?('raise') && record['raise']
-        raise Exception('process_message')
+        raise 'process_message'
       end
       return true
     end
@@ -302,7 +307,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
   def test_retry_error
     records = []
     error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
+    error_records.merge!({0=>true, 4=>true})
     10.times do |i|
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
@@ -386,6 +391,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
               "reason":"unrecognized error"
             }
           }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "9",
+            "status" : 500,
+            "error" : {
+              "type" : "json_parse_exception",
+              "reason":"Invalid UTF-8 start byte 0x92\\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@204fe9c9; line: 1, column: 81]"
+            }
+          }
         }
       ]
     }))
@@ -400,12 +417,12 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
         next unless e.respond_to?(:retry_stream)
         e.retry_stream.each {|time, record| records << record}
       end
-      assert_equal 2, records.length
-      assert_equal 2, records[0]['_id']
-      assert_equal 8, records[1]['_id']
+      assert_equal 2, records.length, "Exp. retry_stream to contain records"
+      assert_equal 2, records[0]['_id'], "Exp record with given ID to in retry_stream"
+      assert_equal 8, records[1]['_id'], "Exp record with given ID to in retry_stream"
       error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 3, error_ids.length
-      assert_equal [5, 6, 7], error_ids
+      assert_equal 4, error_ids.length, "Exp. a certain number of records to be dropped from retry_stream"
+      assert_equal [5, 6, 7, 9], error_ids, "Exp. specific records to be dropped from retry_stream"
       @plugin.error_events.collect {|h| h[:error]}.each do |e|
         assert_true e.respond_to?(:backtrace)
       end

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -19,7 +19,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
   end
 
   def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
-    body ="{\"version\":{\"number\":\"#{version}\"}}"
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
   end
 

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -27,9 +27,15 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     CODE
   end
 
+  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  end
+
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
       to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+    stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
     assert_equal(expected, xpack_info)
   end
@@ -37,18 +43,21 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
       to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+    stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
     stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
       to_return(:status => 404, :body => "", :headers => {})
+    stub_elastic_info
     assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
   end
 
   def test_ilm_policy_exists
     stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info
     assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
   end
 
@@ -59,6 +68,7 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
       to_return(:status => 200, :body => "", :headers => {})
+    stub_elastic_info
     create_ilm_policy("fluent-policy")
 
     assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)

data/test/plugin/test_in_elasticsearch.rb

@@ -31,6 +31,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     @driver ||= Fluent::Test::Driver::Input.new(Fluent::Plugin::ElasticsearchInput).configure(conf)
   end
 
+  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  end
+
   def sample_response(index_name="fluentd")
     {
       "took"=>4,
@@ -322,6 +327,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG)
     driver.run(expect_emits: 1, timeout: 10)
@@ -337,6 +343,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[index_name #{index_name}])
     driver.run(expect_emits: 1, timeout: 10)
@@ -352,6 +359,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp])
     driver.run(expect_emits: 1, timeout: 10)
@@ -370,6 +378,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp true
                       timestamp_key_format %Y-%m-%dT%H:%M:%S.%N%z
@@ -389,6 +398,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[docinfo true])
     driver.run(expect_emits: 1, timeout: 10)
@@ -412,6 +422,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
+    stub_elastic_info
 
     driver(CONFIG + %[num_slices 2])
     driver.run(expect_emits: 1, timeout: 10)
@@ -434,6 +445,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
         body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}") do
       connection += 1
     end
+    stub_elastic_info
     scroll_request.to_return(lambda do |req|
                                if connection <= 1
                                  {status: 200, body: sample_scroll_response_2.to_s,