logstash-output-elasticsearch 11.0.0-java → 11.0.4-java

data/spec/integration/outputs/update_spec.rb

@@ -1,116 +1,114 @@
 require_relative "../../../spec/es_spec_helper"
 
-if ESHelper.es_version_satisfies?(">= 2")
-  describe "Update actions without scripts", :integration => true do
-    require "logstash/outputs/elasticsearch"
-
-    def get_es_output( options={} )
-      settings = {
-        "manage_template" => true,
-        "index" => "logstash-update",
-        "template_overwrite" => true,
-        "hosts" => get_host_port(),
-        "action" => "update"
-      }
-      LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+describe "Update actions without scripts", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "update"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => doc_type,
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
+    @es.indices.refresh
+  end
+
+  it "should fail without a document_id" do
+    subject = get_es_output
+    expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+  end
+
+  context "when update only" do
+    it "should not create new document" do
+      subject = get_es_output({ 'document_id' => "456" } )
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+    end
+
+    it "should update existing document" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+      expect(r["_source"]["message"]).to eq('updated message here')
+    end
+
+    # The es ruby client treats the data field differently. Make sure this doesn't
+    # raise an exception
+    it "should update an existing document that has a 'data' field" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+      expect(r["_source"]["data"]).to eq('updated message here')
+      expect(r["_source"]["message"]).to eq('foo')
+    end
+
+    it "should allow default (internal) version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+      subject.register
     end
 
-    before :each do
-      @es = get_client
-      # Delete all templates first.
-      # Clean ES of data before we start.
-      @es.indices.delete_template(:name => "*")
-      # This can fail if there are no indexes, ignore failure.
-      @es.indices.delete(:index => "*") rescue nil
-      @es.index(
-        :index => 'logstash-update',
-        :type => doc_type,
-        :id => "123",
-        :body => { :message => 'Test', :counter => 1 }
-      )
-      @es.indices.refresh
+    it "should allow internal version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+      subject.register
     end
 
-    it "should fail without a document_id" do
-      subject = get_es_output
+    it "should not allow external version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
-    context "when update only" do
-      it "should not create new document" do
-        subject = get_es_output({ 'document_id' => "456" } )
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-      end
-
-      it "should update existing document" do
-        subject = get_es_output({ 'document_id' => "123" })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["message"]).to eq('updated message here')
-      end
-
-      # The es ruby client treats the data field differently. Make sure this doesn't
-      # raise an exception
-      it "should update an existing document that has a 'data' field" do
-        subject = get_es_output({ 'document_id' => "123" })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["data"]).to eq('updated message here')
-        expect(r["_source"]["message"]).to eq('foo')
-      end
-
-      it "should allow default (internal) version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99" })
-        subject.register
-      end
-
-      it "should allow internal version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
-        subject.register
-      end
-
-      it "should not allow external version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
-
-      it "should not allow external_gt version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
-
-      it "should not allow external_gte version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
+    it "should not allow external_gt version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
 
+    it "should not allow external_gte version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
-    context "when update with upsert" do
-      it "should create new documents with provided upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('upsert message')
-      end
-
-      it "should create new documents with event/doc as upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('sample message here')
-      end
-
-      it "should fail on documents with event/doc as upsert at external version" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
+  end
+
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["message"]).to eq('upsert message')
+    end
+
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["message"]).to eq('sample message here')
+    end
+
+    it "should fail on documents with event/doc as upsert at external version" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,9 +1,5 @@
 require "logstash/devutils/rspec/spec_helper"
 
-unless defined?(LogStash::OSS)
-  LogStash::OSS = ENV['DISTRIBUTION'] != "default"
-end
-
 require "logstash/outputs/elasticsearch"
 
 module LogStash::Outputs::ElasticSearch::SpecHelper
@@ -11,4 +7,4 @@ end
 
 RSpec.configure do |config|
   config.include LogStash::Outputs::ElasticSearch::SpecHelper
-end
+end

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -31,18 +31,12 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
     # end
   end
 
-  @@logstash_oss = LogStash::OSS
-
   before(:each) do
-    change_constant :OSS, false, target: LogStash # assume non-OSS by default
-
     stub_plugin_register! if do_register
   end
 
   after(:each) do
     subject.close if do_register
-
-    change_constant :OSS, @@logstash_oss, target: LogStash
   end
 
   context "default configuration" do
@@ -115,14 +109,6 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       end
     end
 
-    it "uses data-streams on LS 8.0 (OSS)" do
-      change_constant :LOGSTASH_VERSION, '8.0.1' do
-        change_constant :OSS, true, target: LogStash do
-          expect( subject.data_stream_config? ).to be true
-        end
-      end
-    end
-
     context 'old ES' do
 
       let(:es_version) { '7.8.1' }

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -8,15 +8,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
   let(:es_node_versions) { [ "0.0.0" ] }
-  let(:oss) { true }
   let(:license_status) { 'active' }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
 
   let(:manticore_double) { double("manticore a") }
   before(:each) do
-    stub_const('LogStash::OSS', oss)
-
     response_double = double("manticore response").as_null_object
     # Allow healtchecks
     allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
@@ -278,50 +275,46 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       allow(subject).to receive(:health_check_request)
     end
 
-    context "when using default logstash distribution" do
-      let(:oss) { false }
-
-      context "if ES doesn't return a valid license" do
-        let(:license_status) { nil }
+    context "if ES doesn't return a valid license" do
+      let(:license_status) { nil }
 
-        it "marks the url as dead" do
-          subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(0)
-        end
+      it "marks the url as dead" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(0)
+      end
 
-        it "logs a warning" do
-          expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
-          subject.update_initial_urls
-        end
+      it "logs a warning" do
+        expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
+        subject.update_initial_urls
       end
+    end
 
-      context "if ES returns a valid license" do
-        let(:license_status) { 'active' }
+    context "if ES returns a valid license" do
+      let(:license_status) { 'active' }
 
-        it "marks the url as active" do
-          subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
-        end
+      it "marks the url as active" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(1)
+      end
 
-        it "does not log a warning" do
-          expect(subject.license_checker).to_not receive(:warn_no_license)
-          expect(subject.license_checker).to_not receive(:warn_invalid_license)
-          subject.update_initial_urls
-        end
+      it "does not log a warning" do
+        expect(subject.license_checker).to_not receive(:warn_no_license)
+        expect(subject.license_checker).to_not receive(:warn_invalid_license)
+        subject.update_initial_urls
       end
+    end
 
-      context "if ES returns an invalid license" do
-        let(:license_status) { 'invalid' }
+    context "if ES returns an invalid license" do
+      let(:license_status) { 'invalid' }
 
-        it "marks the url as active" do
-          subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
-        end
+      it "marks the url as active" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(1)
+      end
 
-        it "logs a warning" do
-          expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
-          subject.update_initial_urls
-        end
+      it "logs a warning" do
+        expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
+        subject.update_initial_urls
       end
     end
   end

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -4,19 +4,19 @@ require "logstash/outputs/elasticsearch/template_manager"
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
   describe ".default_template_path" do
-    context "elasticsearch 1.x" do
-      it "chooses the 2x template" do
-        expect(described_class.default_template_path(1)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
+    context "elasticsearch 6.x" do
+      it "chooses the 6x template" do
+        expect(described_class.default_template_path(6)).to end_with("/templates/ecs-disabled/elasticsearch-6x.json")
       end
     end
-    context "elasticsearch 2.x" do
-      it "chooses the 2x template" do
-        expect(described_class.default_template_path(2)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
+    context "elasticsearch 7.x" do
+      it "chooses the 7x template" do
+        expect(described_class.default_template_path(7)).to end_with("/templates/ecs-disabled/elasticsearch-7x.json")
       end
     end
-    context "elasticsearch 5.x" do
-      it "chooses the 5x template" do
-        expect(described_class.default_template_path(5)).to end_with("/templates/ecs-disabled/elasticsearch-5x.json")
+    context "elasticsearch 8.x" do
+      it "chooses the 8x template" do
+        expect(described_class.default_template_path(8)).to end_with("/templates/ecs-disabled/elasticsearch-8x.json")
       end
     end
   end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -7,7 +7,7 @@ require "logstash/outputs/elasticsearch"
 describe LogStash::Outputs::ElasticSearch do
   subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
-  let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample }
+  let(:maximum_seen_major_version) { [6,7,8].sample }
 
   let(:do_register) { true }
 
@@ -79,13 +79,6 @@ describe LogStash::Outputs::ElasticSearch do
             expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
           end
         end
-
-        context "for < 6.0 elasticsearch clusters" do
-          let(:maximum_seen_major_version) { 5 }
-          it "should get the type from the event" do
-            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
-          end
-        end
       end
 
       context "with 'document type set'" do
@@ -343,7 +336,7 @@ describe LogStash::Outputs::ElasticSearch do
                               }
                     }]
         }
-    end
+      end
 
       before(:each) do
         allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
@@ -771,9 +764,9 @@ describe LogStash::Outputs::ElasticSearch do
 
         context 'when getting any other exception' do
           it 'should log at WARN level' do
-            dlog = double_logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, dlog)
-            expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            logger = double("logger").as_null_object
+            subject.instance_variable_set(:@logger, logger)
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
             subject.handle_dlq_status("Could not index event to Elasticsearch.",
               [:action, :params, :event], :some_status, mock_response)
@@ -782,9 +775,9 @@ describe LogStash::Outputs::ElasticSearch do
 
         context 'when the response does not include [error]' do
           it 'should not fail, but just log a warning' do
-            dlog = double_logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, dlog)
-            expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            logger = double("logger").as_null_object
+            subject.instance_variable_set(:@logger, logger)
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
             mock_response = { 'index' => {} }
             expect do
               subject.handle_dlq_status("Could not index event to Elasticsearch.",
@@ -804,12 +797,55 @@ describe LogStash::Outputs::ElasticSearch do
       # We should still log when sending to the DLQ.
       # This shall be solved by another issue, however: logstash-output-elasticsearch#772
       it 'should send the event to the DLQ instead, and not log' do
-        expect(dlq_writer).to receive(:write).once.with(:event, /Could not index/)
+        event = LogStash::Event.new("foo" => "bar")
+        expect(dlq_writer).to receive(:write).once.with(event, /Could not index/)
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
-        subject.handle_dlq_status("Could not index event to Elasticsearch.",
-          [:action, :params, :event], :some_status, mock_response)
+        action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
+        subject.handle_dlq_status("Could not index event to Elasticsearch.", action, 404, mock_response)
       end
     end
+
+    context 'with response status 400' do
+
+      let(:options) { super().merge 'document_id' => '%{foo}' }
+
+      let(:events) { [ LogStash::Event.new("foo" => "bar") ] }
+
+      let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
+
+      let(:bulk_response) do
+        {
+            "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
+            [{
+                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status"=>400,
+                           "error"=>{"type" => "illegal_argument_exception", "reason" => "TEST" }
+                  }
+            }]
+        }
+      end
+
+      before(:each) do
+        allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
+      end
+
+      it "should write event to DLQ" do
+        expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
+          expect( args.size ).to eql 2
+
+          event, reason = *args
+          expect( event ).to be_a LogStash::Event
+          expect( event ).to be events.first
+          expect( reason ).to start_with 'Could not index event to Elasticsearch. status: 400, action: ["index"'
+          expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
+
+          method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
+        end.once
+
+        event_action_tuples = subject.map_events(events)
+        subject.send(:submit, event_action_tuples)
+      end
+
+    end if LOGSTASH_VERSION > '7.0'
   end
 
   describe "custom headers" do