logstash-output-elasticsearch 10.8.6-java → 11.0.0-java

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -1,5 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
+require 'cabin'
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
   let(:logger) { Cabin::Channel.get }

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -1,6 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
-require "json"
+require 'cabin'
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
@@ -9,7 +9,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
   let(:es_node_versions) { [ "0.0.0" ] }
   let(:oss) { true }
-  let(:valid_license) { true }
+  let(:license_status) { 'active' }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
 
@@ -26,8 +26,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
     allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
-    allow(subject.license_checker).to receive(:oss?).and_return(oss)
-    allow(subject.license_checker).to receive(:valid_es_license?).and_return(valid_license)
+    allow(subject.license_checker).to receive(:license_status).and_return(license_status)
   end
 
   after do
@@ -283,21 +282,21 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       let(:oss) { false }
 
       context "if ES doesn't return a valid license" do
-        let(:valid_license) { false }
+        let(:license_status) { nil }
 
-        it "marks the url as active" do
+        it "marks the url as dead" do
           subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
+          expect(subject.alive_urls_count).to eq(0)
         end
 
         it "logs a warning" do
-          expect(subject.license_checker).to receive(:log_license_deprecation_warn).once
+          expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
           subject.update_initial_urls
         end
       end
 
       context "if ES returns a valid license" do
-        let(:valid_license) { true }
+        let(:license_status) { 'active' }
 
         it "marks the url as active" do
           subject.update_initial_urls
@@ -305,7 +304,22 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         end
 
         it "does not log a warning" do
-          expect(subject.license_checker).to_not receive(:log_license_deprecation_warn)
+          expect(subject.license_checker).to_not receive(:warn_no_license)
+          expect(subject.license_checker).to_not receive(:warn_invalid_license)
+          subject.update_initial_urls
+        end
+      end
+
+      context "if ES returns an invalid license" do
+        let(:license_status) { 'invalid' }
+
+        it "marks the url as active" do
+          subject.update_initial_urls
+          expect(subject.alive_urls_count).to eq(1)
+        end
+
+        it "logs a warning" do
+          expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
           subject.update_initial_urls
         end
       end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -1,7 +1,6 @@
-require_relative "../../../../spec/es_spec_helper"
-require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../../spec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
-require "java"
+require "cabin"
 
 describe LogStash::Outputs::ElasticSearch::HttpClient do
   let(:ssl) { nil }

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -1,7 +1,5 @@
 require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/elasticsearch/http_client"
-require "java"
-require "json"
+require "logstash/outputs/elasticsearch/template_manager"
 
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
 

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -1,6 +1,5 @@
-require_relative "../../../spec/es_spec_helper"
+require_relative "../../../spec/spec_helper"
 require 'stud/temporary'
-require "logstash/outputs/elasticsearch"
 require 'manticore/client'
 
 describe "Proxy option" do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1,6 +1,7 @@
-require_relative "../../../spec/es_spec_helper"
+require_relative "../../../spec/spec_helper"
 require "base64"
 require "flores/random"
+require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/elasticsearch"
 
 describe LogStash::Outputs::ElasticSearch do
@@ -10,18 +11,30 @@ describe LogStash::Outputs::ElasticSearch do
 
   let(:do_register) { true }
 
+  let(:stub_http_client_pool!) do
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:start)
+  end
+
+  let(:after_successful_connection_thread_mock) do
+    double('after_successful_connection_thread', value: true)
+  end
+
   before(:each) do
     if do_register
-      # Build the client and set mocks before calling register to avoid races.
-      subject.build_client
+      stub_http_client_pool!
+
+      allow(subject).to receive(:finish_register) # stub-out thread completion (to avoid error log entries)
+
+      # emulate 'successful' ES connection on the same thread
+      allow(subject).to receive(:after_successful_connection) { |&block| block.call }.
+          and_return after_successful_connection_thread_mock
+      allow(subject).to receive(:stop_after_successful_connection_thread)
+
+      subject.register
 
-      # Rspec mocks can't handle background threads, so... we can't use any
-      allow(subject.client.pool).to receive(:start_resurrectionist)
-      allow(subject.client.pool).to receive(:start_sniffer)
-      allow(subject.client.pool).to receive(:healthcheck!)
       allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
       allow(subject.client).to receive(:get_xpack_info)
-      subject.register
+
       subject.client.pool.adapter.manticore.respond_with(:body => "{}")
     end
   end
@@ -44,6 +57,12 @@ describe LogStash::Outputs::ElasticSearch do
     let(:manticore_urls) { subject.client.pool.urls }
     let(:manticore_url) { manticore_urls.first }
 
+    let(:stub_http_client_pool!) do
+      [:start_resurrectionist, :start_sniffer, :healthcheck!].each do |method|
+        allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(method)
+      end
+    end
+
     describe "getting a document type" do
       context "if document_type isn't set" do
         let(:options) { super().merge("document_type" => nil)}
@@ -253,8 +272,7 @@ describe LogStash::Outputs::ElasticSearch do
       before do
         allow(subject).to receive(:retrying_submit).with(anything)
         events.each_with_index do |e,i|
-          et = events_tuples[i]
-          allow(subject).to receive(:event_action_tuple).with(e).and_return(et)
+          allow(subject).to receive(:event_action_tuple).with(e).and_return(events_tuples[i])
         end
         subject.multi_receive(events)
       end
@@ -345,7 +363,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       it "should log specific error message" do
         expect(subject.logger).to receive(:error).with(/Encountered an unexpected error/i,
-                                                       hash_including(:error_message => 'Sent 2 documents but Elasticsearch returned 3 responses (likely a bug with _bulk endpoint)'))
+                                                       hash_including(:message => 'Sent 2 documents but Elasticsearch returned 3 responses (likely a bug with _bulk endpoint)'))
 
         subject.multi_receive(events)
       end
@@ -402,7 +420,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     before do
       # Expect a timeout to be logged.
-      expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request to Elasticsearch/i, anything).at_least(:once)
+      expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request/i, anything).at_least(:once)
       expect(subject.client).to receive(:bulk).at_least(:twice).and_call_original
     end
 
@@ -416,13 +434,14 @@ describe LogStash::Outputs::ElasticSearch do
   end
 
   describe "the action option" do
+
     context "with a sprintf action" do
       let(:options) { {"action" => "%{myactionfield}" } }
 
       let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
 
       it "should interpolate the requested action value when creating an event_action_tuple" do
-        expect(subject.event_action_tuple(event).first).to eql("update")
+        expect(subject.send(:event_action_tuple, event).first).to eql("update")
       end
     end
 
@@ -432,7 +451,7 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
 
       it "should obtain specific action's params from event_action_tuple" do
-        expect(subject.event_action_tuple(event)[1]).to include(:_upsert)
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:_upsert)
       end
     end
 
@@ -440,6 +459,8 @@ describe LogStash::Outputs::ElasticSearch do
       let(:options) { {"action" => "SOME Garbaaage"} }
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
 
+      before { allow(subject).to receive(:finish_register) }
+
       it "should raise a configuration error" do
         expect { subject.register }.to raise_error(LogStash::ConfigurationError)
       end
@@ -447,13 +468,14 @@ describe LogStash::Outputs::ElasticSearch do
   end
 
   describe "the pipeline option" do
+
     context "with a sprintf and set pipeline" do
       let(:options) { {"pipeline" => "%{pipeline}" } }
 
       let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
 
       it "should interpolate the pipeline value and set it" do
-        expect(subject.event_action_tuple(event)[1]).to include(:pipeline => "my-ingest-pipeline")
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
       end
     end
 
@@ -463,7 +485,7 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event) { LogStash::Event.new("pipeline" => "") }
 
       it "should interpolate the pipeline value but not set it because it is empty" do
-        expect(subject.event_action_tuple(event)[1]).not_to include(:pipeline)
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
       end
     end
   end
@@ -505,8 +527,8 @@ describe LogStash::Outputs::ElasticSearch do
 
       it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do
         allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).not_to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).not_to include({subject.send(:retry_on_conflict_action_name) => num_retries})
       end
     end
 
@@ -514,8 +536,8 @@ describe LogStash::Outputs::ElasticSearch do
       let(:options) { super().merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).to include({subject.send(:retry_on_conflict_action_name) => num_retries})
       end
     end
 
@@ -523,8 +545,8 @@ describe LogStash::Outputs::ElasticSearch do
       let(:options) { super().merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) }
 
       it "should set the retry_on_conflict parameter when creating an event_action_tuple" do
-        action, params, event_data = subject.event_action_tuple(event)
-        expect(params).to include({subject.retry_on_conflict_action_name => num_retries})
+        action, params, event_data = subject.send(:event_action_tuple, event)
+        expect(params).to include({subject.send(:retry_on_conflict_action_name) => num_retries})
         expect(action).to eq("update")
       end
     end
@@ -554,6 +576,8 @@ describe LogStash::Outputs::ElasticSearch do
     let(:do_register) { false }
 
     before :each do
+      allow(subject).to receive(:finish_register)
+
       allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
     end
 
@@ -577,6 +601,12 @@ describe LogStash::Outputs::ElasticSearch do
     let(:custom_parameters_hash) { { "id" => 1, "name" => "logstash" } }
     let(:custom_parameters_query) { custom_parameters_hash.map {|k,v| "#{k}=#{v}" }.join("&") }
 
+    let(:stub_http_client_pool!) do
+      [:start_resurrectionist, :start_sniffer, :healthcheck!].each do |method|
+        allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(method)
+      end
+    end
+
     context "using non-url hosts" do
 
       let(:options) {
@@ -783,7 +813,7 @@ describe LogStash::Outputs::ElasticSearch do
   end
 
   describe "custom headers" do
-    let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) } 
+    let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) }
 
     context "when set" do
       let(:headers) { { "X-Thing" => "Test" } }
@@ -856,6 +886,75 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "post-register ES setup" do
+    let(:do_register) { false }
+    let(:es_version) { '7.10.0' } # DS default on LS 8.x
+    let(:options) { { 'hosts' => '127.0.0.1:9999' } }
+    let(:logger) { subject.logger }
+
+    before do
+      allow(logger).to receive(:error) # expect tracking
+
+      allow(subject).to receive(:last_es_version).and_return es_version
+      # make successful_connection? return true:
+      allow(subject).to receive(:maximum_seen_major_version).and_return Integer(es_version.split('.').first)
+      allow(subject).to receive(:stop_after_successful_connection_thread)
+    end
+
+    it "logs inability to retrieve uuid" do
+      allow(subject).to receive(:install_template)
+      allow(subject).to receive(:ilm_in_use?).and_return nil
+      subject.register
+      subject.send :wait_for_successful_connection
+
+      expect(logger).to have_received(:error).with(/Unable to retrieve Elasticsearch cluster uuid/i, anything)
+    end if LOGSTASH_VERSION >= '7.0.0'
+
+    it "logs template install failure" do
+      allow(subject).to receive(:discover_cluster_uuid)
+      allow(subject).to receive(:ilm_in_use?).and_return nil
+      subject.register
+      subject.send :wait_for_successful_connection
+
+      expect(logger).to have_received(:error).with(/Failed to install template/i, anything)
+    end
+
+    context 'error raised' do
+
+      let(:es_version) { '7.8.0' }
+      let(:options) { super().merge('data_stream' => 'true') }
+      let(:latch) { Concurrent::CountDownLatch.new }
+
+      before do
+        allow(subject).to receive(:install_template)
+        allow(subject).to receive(:discover_cluster_uuid)
+        allow(subject).to receive(:ilm_in_use?).and_return nil
+        # executes from the after_successful_connection thread :
+        allow(subject).to receive(:finish_register) { latch.wait }.and_call_original
+        subject.register
+      end
+
+      it 'keeps logging on multi_receive' do
+        allow(subject).to receive(:retrying_submit)
+        latch.count_down; sleep(1.0)
+
+        expect_logged_error = lambda do |count|
+          expect(logger).to have_received(:error).with(
+              /Elasticsearch setup did not complete normally, please review previously logged errors/i,
+              hash_including(message:  'A data_stream configuration is only supported since Elasticsearch 7.9.0 (detected version 7.8.0), please upgrade your cluster')
+          ).exactly(count).times
+        end
+
+        subject.multi_receive [ LogStash::Event.new('foo' => 1) ]
+        expect_logged_error.call(1)
+
+        subject.multi_receive [ LogStash::Event.new('foo' => 2) ]
+        expect_logged_error.call(2)
+      end
+
+    end
+  end
+
   @private
 
   def stub_manticore_client!(manticore_double = nil)

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -1,6 +1,5 @@
-require_relative "../../../spec/es_spec_helper"
+require_relative "../../../spec/spec_helper"
 require 'stud/temporary'
-require "logstash/outputs/elasticsearch"
 
 describe "SSL option" do
   let(:manticore_double) { double("manticoreSSL #{self.inspect}") }

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -12,6 +12,7 @@ describe "whitelisting error types in expected behavior" do
   before :each do
     allow(subject.logger).to receive(:warn)
     allow(subject).to receive(:maximum_seen_major_version).and_return(0)
+    allow(subject).to receive(:finish_register)
 
     subject.register
 
@@ -39,7 +40,7 @@ describe "whitelisting error types in expected behavior" do
 
   describe "when failure logging is enabled for everything" do
     it "should log a failure on the action" do
-      expect(subject.logger).to have_received(:warn).with("Failed action.", anything)
+      expect(subject.logger).to have_received(:warn).with("Failed action", anything)
     end
   end
 
@@ -47,7 +48,7 @@ describe "whitelisting error types in expected behavior" do
     let(:settings) { super().merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
 
     it "should log a failure on the action" do
-      expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything)
+      expect(subject.logger).not_to have_received(:warn).with("Failed action", anything)
     end
   end
 

data/spec/unit/outputs/license_check_spec.rb

@@ -14,22 +14,6 @@ describe LogStash::Outputs::ElasticSearch::LicenseChecker do
     end
   end
 
-  context "LicenseChecker API required by Pool specs" do
-    subject { described_class }
-
-    it "defines the oss? method" do
-      expect(subject.instance_methods).to include(:oss?)
-    end
-
-    it "defines the valid_es_license? method" do
-      expect(subject.instance_methods).to include(:valid_es_license?)
-    end
-
-    it "defines the log_license_deprecation_warn method" do
-      expect(subject.instance_methods).to include(:log_license_deprecation_warn)
-    end
-  end
-
   context "Pool class API required by the LicenseChecker" do
     subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool }