logstash-output-elasticsearch 10.8.6-java → 11.0.3-java

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -1,5 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
+require 'cabin'
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
   let(:logger) { Cabin::Channel.get }

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -1,6 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
-require "json"
+require 'cabin'
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
@@ -8,15 +8,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
   let(:es_node_versions) { [ "0.0.0" ] }
-  let(:oss) { true }
-  let(:valid_license) { true }
+  let(:license_status) { 'active' }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
 
   let(:manticore_double) { double("manticore a") }
   before(:each) do
-    stub_const('LogStash::OSS', oss)
-
     response_double = double("manticore response").as_null_object
     # Allow healtchecks
     allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
@@ -26,8 +23,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
     allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
-    allow(subject.license_checker).to receive(:oss?).and_return(oss)
-    allow(subject.license_checker).to receive(:valid_es_license?).and_return(valid_license)
+    allow(subject.license_checker).to receive(:license_status).and_return(license_status)
   end
 
   after do
@@ -279,35 +275,46 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       allow(subject).to receive(:health_check_request)
     end
 
-    context "when using default logstash distribution" do
-      let(:oss) { false }
+    context "if ES doesn't return a valid license" do
+      let(:license_status) { nil }
 
-      context "if ES doesn't return a valid license" do
-        let(:valid_license) { false }
+      it "marks the url as dead" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(0)
+      end
 
-        it "marks the url as active" do
-          subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
-        end
+      it "logs a warning" do
+        expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
+        subject.update_initial_urls
+      end
+    end
 
-        it "logs a warning" do
-          expect(subject.license_checker).to receive(:log_license_deprecation_warn).once
-          subject.update_initial_urls
-        end
+    context "if ES returns a valid license" do
+      let(:license_status) { 'active' }
+
+      it "marks the url as active" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(1)
       end
 
-      context "if ES returns a valid license" do
-        let(:valid_license) { true }
+      it "does not log a warning" do
+        expect(subject.license_checker).to_not receive(:warn_no_license)
+        expect(subject.license_checker).to_not receive(:warn_invalid_license)
+        subject.update_initial_urls
+      end
+    end
 
-        it "marks the url as active" do
-          subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
-        end
+    context "if ES returns an invalid license" do
+      let(:license_status) { 'invalid' }
 
-        it "does not log a warning" do
-          expect(subject.license_checker).to_not receive(:log_license_deprecation_warn)
-          subject.update_initial_urls
-        end
+      it "marks the url as active" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(1)
+      end
+
+      it "logs a warning" do
+        expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
+        subject.update_initial_urls
       end
     end
   end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -1,7 +1,6 @@
-require_relative "../../../../spec/es_spec_helper"
-require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../../spec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
-require "java"
+require "cabin"
 
 describe LogStash::Outputs::ElasticSearch::HttpClient do
   let(:ssl) { nil }

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -1,24 +1,22 @@
 require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/elasticsearch/http_client"
-require "java"
-require "json"
+require "logstash/outputs/elasticsearch/template_manager"
 
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
   describe ".default_template_path" do
-    context "elasticsearch 1.x" do
-      it "chooses the 2x template" do
-        expect(described_class.default_template_path(1)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
+    context "elasticsearch 6.x" do
+      it "chooses the 6x template" do
+        expect(described_class.default_template_path(6)).to end_with("/templates/ecs-disabled/elasticsearch-6x.json")
       end
     end
-    context "elasticsearch 2.x" do
-      it "chooses the 2x template" do
-        expect(described_class.default_template_path(2)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
+    context "elasticsearch 7.x" do
+      it "chooses the 7x template" do
+        expect(described_class.default_template_path(7)).to end_with("/templates/ecs-disabled/elasticsearch-7x.json")
       end
     end
-    context "elasticsearch 5.x" do
-      it "chooses the 5x template" do
-        expect(described_class.default_template_path(5)).to end_with("/templates/ecs-disabled/elasticsearch-5x.json")
+    context "elasticsearch 8.x" do
+      it "chooses the 8x template" do
+        expect(described_class.default_template_path(8)).to end_with("/templates/ecs-disabled/elasticsearch-8x.json")
       end
     end
   end

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -1,6 +1,5 @@
-require_relative "../../../spec/es_spec_helper"
+require_relative "../../../spec/spec_helper"
 require 'stud/temporary'
-require "logstash/outputs/elasticsearch"
 require 'manticore/client'
 
 describe "Proxy option" do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1,27 +1,40 @@
-require_relative "../../../spec/es_spec_helper"
+require_relative "../../../spec/spec_helper"
 require "base64"
 require "flores/random"
+require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/elasticsearch"
 
 describe LogStash::Outputs::ElasticSearch do
   subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
-  let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample }
+  let(:maximum_seen_major_version) { [6,7,8].sample }
 
   let(:do_register) { true }
 
+  let(:stub_http_client_pool!) do
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:start)
+  end
+
+  let(:after_successful_connection_thread_mock) do
+    double('after_successful_connection_thread', value: true)
+  end
+
   before(:each) do
     if do_register
-      # Build the client and set mocks before calling register to avoid races.
-      subject.build_client
+      stub_http_client_pool!
+
+      allow(subject).to receive(:finish_register) # stub-out thread completion (to avoid error log entries)
+
+      # emulate 'successful' ES connection on the same thread
+      allow(subject).to receive(:after_successful_connection) { |&block| block.call }.
+          and_return after_successful_connection_thread_mock
+      allow(subject).to receive(:stop_after_successful_connection_thread)
+
+      subject.register
 
-      # Rspec mocks can't handle background threads, so... we can't use any
-      allow(subject.client.pool).to receive(:start_resurrectionist)
-      allow(subject.client.pool).to receive(:start_sniffer)
-      allow(subject.client.pool).to receive(:healthcheck!)
       allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
       allow(subject.client).to receive(:get_xpack_info)
-      subject.register
+
       subject.client.pool.adapter.manticore.respond_with(:body => "{}")
     end
   end
@@ -44,6 +57,12 @@ describe LogStash::Outputs::ElasticSearch do
     let(:manticore_urls) { subject.client.pool.urls }
     let(:manticore_url) { manticore_urls.first }
 
+    let(:stub_http_client_pool!) do
+      [:start_resurrectionist, :start_sniffer, :healthcheck!].each do |method|
+        allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(method)
+      end
+    end
+
     describe "getting a document type" do
       context "if document_type isn't set" do
         let(:options) { super().merge("document_type" => nil)}
@@ -60,13 +79,6 @@ describe LogStash::Outputs::ElasticSearch do
             expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
           end
         end
-
-        context "for < 6.0 elasticsearch clusters" do
-          let(:maximum_seen_major_version) { 5 }
-          it "should get the type from the event" do
-            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
-          end
-        end
       end
 
       context "with 'document type set'" do
@@ -253,8 +265,7 @@ describe LogStash::Outputs::ElasticSearch do
       before do
         allow(subject).to receive(:retrying_submit).with(anything)
         events.each_with_index do |e,i|
-          et = events_tuples[i]
-          allow(subject).to receive(:event_action_tuple).with(e).and_return(et)
+          allow(subject).to receive(:event_action_tuple).with(e).and_return(events_tuples[i])
         end
         subject.multi_receive(events)
       end
@@ -325,7 +336,7 @@ describe LogStash::Outputs::ElasticSearch do
                               }
                     }]
         }
-    end
+      end
 
       before(:each) do
         allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
@@ -345,7 +356,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       it "should log specific error message" do
         expect(subject.logger).to receive(:error).with(/Encountered an unexpected error/i,
-                                                       hash_including(:error_message => 'Sent 2 documents but Elasticsearch returned 3 responses (likely a bug with _bulk endpoint)'))
+                                                       hash_including(:message => 'Sent 2 documents but Elasticsearch returned 3 responses (likely a bug with _bulk endpoint)'))
 
         subject.multi_receive(events)
       end
@@ -402,7 +413,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     before do
       # Expect a timeout to be logged.
-      expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request to Elasticsearch/i, anything).at_least(:once)
+      expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request/i, anything).at_least(:once)
       expect(subject.client).to receive(:bulk).at_least(:twice).and_call_original
     end
 
@@ -416,13 +427,14 @@ describe LogStash::Outputs::ElasticSearch do
   end
 
   describe "the action option" do
+
     context "with a sprintf action" do
       let(:options) { {"action" => "%{myactionfield}" } }
 
       let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
 
       it "should interpolate the requested action value when creating an event_action_tuple" do
-        expect(subject.event_action_tuple(event).first).to eql("update")
+        expect(subject.send(:event_action_tuple, event).first).to eql("update")
       end
     end
 
@@ -432,7 +444,7 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
 
       it "should obtain specific action's params from event_action_tuple" do
-        expect(subject.event_action_tuple(event)[1]).to include(:_upsert)
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:_upsert)
       end
     end
 
@@ -440,6 +452,8 @@ describe LogStash::Outputs::ElasticSearch do
       let(:options) { {"action" => "SOME Garbaaage"} }
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
 
+      before { allow(subject).to receive(:finish_register) }
+
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error(LogStash::ConfigurationError)
       end
@@ -447,13 +461,14 @@ describe LogStash::Outputs::ElasticSearch do
   end
 
   describe "the pipeline option" do
+
     context "with a sprintf and set pipeline" do
       let(:options) { {"pipeline" => "%{pipeline}" } }
 
       let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
 
       it "should interpolate the pipeline value and set it" do
-        expect(subject.event_action_tuple(event)[1]).to include(:pipeline => "my-ingest-pipeline")
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
       end
     end
 
@@ -463,7 +478,7 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { LogStash::Event.new("pipeline" => "") }
 
       it "should interpolate the pipeline value but not set it because it is empty" do
-        expect(subject.event_action_tuple(event)[1]).not_to include(:pipeline)
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
       end
     end
   end
@@ -505,8 +520,8 @@ describe LogStash::Outputs::ElasticSearch do
 
       it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do
         allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).not_to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).not_to include({subject.send(:retry_on_conflict_action_name) => num_retries})
       end
     end
 
@@ -514,8 +529,8 @@ describe LogStash::Outputs::ElasticSearch do
       let(:options) { super().merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).to include({subject.send(:retry_on_conflict_action_name) => num_retries})
       end
     end
 
@@ -523,8 +538,8 @@ describe LogStash::Outputs::ElasticSearch do
       let(:options) { super().merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).to include({subject.send(:retry_on_conflict_action_name) => num_retries})
         expect(action).to eq("update")
       end
     end
@@ -554,6 +569,8 @@ describe LogStash::Outputs::ElasticSearch do
     let(:do_register) { false }
 
     before :each do
+      allow(subject).to receive(:finish_register)
+
       allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
     end
 
@@ -577,6 +594,12 @@ describe LogStash::Outputs::ElasticSearch do
     let(:custom_parameters_hash) { { "id" => 1, "name" => "logstash" } }
     let(:custom_parameters_query) { custom_parameters_hash.map {|k,v| "#{k}=#{v}" }.join("&") }
 
+    let(:stub_http_client_pool!) do
+      [:start_resurrectionist, :start_sniffer, :healthcheck!].each do |method|
+        allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(method)
+      end
+    end
+
     context "using non-url hosts" do
 
       let(:options) {
@@ -741,9 +764,9 @@ describe LogStash::Outputs::ElasticSearch do
 
         context 'when getting any other exception' do
           it 'should log at WARN level' do
-            dlog = double_logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, dlog)
-            expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            logger = double("logger").as_null_object
+            subject.instance_variable_set(:@logger, logger)
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
             subject.handle_dlq_status("Could not index event to Elasticsearch.",
               [:action, :params, :event], :some_status, mock_response)
@@ -752,9 +775,9 @@ describe LogStash::Outputs::ElasticSearch do
 
         context 'when the response does not include [error]' do
           it 'should not fail, but just log a warning' do
-            dlog = double_logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, dlog)
-            expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            logger = double("logger").as_null_object
+            subject.instance_variable_set(:@logger, logger)
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
             mock_response = { 'index' => {} }
             expect do
               subject.handle_dlq_status("Could not index event to Elasticsearch.",
@@ -774,16 +797,59 @@ describe LogStash::Outputs::ElasticSearch do
       # We should still log when sending to the DLQ.
       # This shall be solved by another issue, however: logstash-output-elasticsearch#772
       it 'should send the event to the DLQ instead, and not log' do
-        expect(dlq_writer).to receive(:write).once.with(:event, /Could not index/)
+        event = LogStash::Event.new("foo" => "bar")
+        expect(dlq_writer).to receive(:write).once.with(event, /Could not index/)
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
-        subject.handle_dlq_status("Could not index event to Elasticsearch.",
-          [:action, :params, :event], :some_status, mock_response)
+        action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
+        subject.handle_dlq_status("Could not index event to Elasticsearch.", action, 404, mock_response)
       end
     end
+
+    context 'with response status 400' do
+
+      let(:options) { super().merge 'document_id' => '%{foo}' }
+
+      let(:events) { [ LogStash::Event.new("foo" => "bar") ] }
+
+      let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
+
+      let(:bulk_response) do
+        {
+            "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
+            [{
+                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status"=>400,
+                           "error"=>{"type" => "illegal_argument_exception", "reason" => "TEST" }
+                  }
+            }]
+        }
+      end
+
+      before(:each) do
+        allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
+      end
+
+      it "should write event to DLQ" do
+        expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
+          expect( args.size ).to eql 2
+
+          event, reason = *args
+          expect( event ).to be_a LogStash::Event
+          expect( event ).to be events.first
+          expect( reason ).to start_with 'Could not index event to Elasticsearch. status: 400, action: ["index"'
+          expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
+
+          method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
+        end.once
+
+        event_action_tuples = subject.map_events(events)
+        subject.send(:submit, event_action_tuples)
+      end
+
+    end if LOGSTASH_VERSION > '7.0'
   end
 
   describe "custom headers" do
-    let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) } 
+    let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) }
 
     context "when set" do
       let(:headers) { { "X-Thing" => "Test" } }
@@ -856,6 +922,75 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "post-register ES setup" do
+    let(:do_register) { false }
+    let(:es_version) { '7.10.0' } # DS default on LS 8.x
+    let(:options) { { 'hosts' => '127.0.0.1:9999' } }
+    let(:logger) { subject.logger }
+
+    before do
+      allow(logger).to receive(:error) # expect tracking
+
+      allow(subject).to receive(:last_es_version).and_return es_version
+      # make successful_connection? return true:
+      allow(subject).to receive(:maximum_seen_major_version).and_return Integer(es_version.split('.').first)
+      allow(subject).to receive(:stop_after_successful_connection_thread)
+    end
+
+    it "logs inability to retrieve uuid" do
+      allow(subject).to receive(:install_template)
+      allow(subject).to receive(:ilm_in_use?).and_return nil
+      subject.register
+      subject.send :wait_for_successful_connection
+
+      expect(logger).to have_received(:error).with(/Unable to retrieve Elasticsearch cluster uuid/i, anything)
+    end if LOGSTASH_VERSION >= '7.0.0'
+
+    it "logs template install failure" do
+      allow(subject).to receive(:discover_cluster_uuid)
+      allow(subject).to receive(:ilm_in_use?).and_return nil
+      subject.register
+      subject.send :wait_for_successful_connection
+
+      expect(logger).to have_received(:error).with(/Failed to install template/i, anything)
+    end
+
+    context 'error raised' do
+
+      let(:es_version) { '7.8.0' }
+      let(:options) { super().merge('data_stream' => 'true') }
+      let(:latch) { Concurrent::CountDownLatch.new }
+
+      before do
+        allow(subject).to receive(:install_template)
+        allow(subject).to receive(:discover_cluster_uuid)
+        allow(subject).to receive(:ilm_in_use?).and_return nil
+        # executes from the after_successful_connection thread :
+        allow(subject).to receive(:finish_register) { latch.wait }.and_call_original
+        subject.register
+      end
+
+      it 'keeps logging on multi_receive' do
+        allow(subject).to receive(:retrying_submit)
+        latch.count_down; sleep(1.0)
+
+        expect_logged_error = lambda do |count|
+          expect(logger).to have_received(:error).with(
+              /Elasticsearch setup did not complete normally, please review previously logged errors/i,
+              hash_including(message:  'A data_stream configuration is only supported since Elasticsearch 7.9.0 (detected version 7.8.0), please upgrade your cluster')
+          ).exactly(count).times
+        end
+
+        subject.multi_receive [ LogStash::Event.new('foo' => 1) ]
+        expect_logged_error.call(1)
+
+        subject.multi_receive [ LogStash::Event.new('foo' => 2) ]
+        expect_logged_error.call(2)
+      end
+
+    end
+  end
+
   @private
 
   def stub_manticore_client!(manticore_double = nil)