logstash-output-elasticsearch 11.0.1-java → 11.0.5-java

data/spec/integration/outputs/update_spec.rb

@@ -1,116 +1,114 @@
 require_relative "../../../spec/es_spec_helper"
 
-if ESHelper.es_version_satisfies?(">= 2")
-  describe "Update actions without scripts", :integration => true do
-    require "logstash/outputs/elasticsearch"
-
-    def get_es_output( options={} )
-      settings = {
-        "manage_template" => true,
-        "index" => "logstash-update",
-        "template_overwrite" => true,
-        "hosts" => get_host_port(),
-        "action" => "update"
-      }
-      LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+describe "Update actions without scripts", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "update"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => doc_type,
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
+    @es.indices.refresh
+  end
+
+  it "should fail without a document_id" do
+    subject = get_es_output
+    expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+  end
+
+  context "when update only" do
+    it "should not create new document" do
+      subject = get_es_output({ 'document_id' => "456" } )
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+    end
+
+    it "should update existing document" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+      expect(r["_source"]["message"]).to eq('updated message here')
+    end
+
+    # The es ruby client treats the data field differently. Make sure this doesn't
+    # raise an exception
+    it "should update an existing document that has a 'data' field" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+      expect(r["_source"]["data"]).to eq('updated message here')
+      expect(r["_source"]["message"]).to eq('foo')
+    end
+
+    it "should allow default (internal) version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+      subject.register
     end
 
-    before :each do
-      @es = get_client
-      # Delete all templates first.
-      # Clean ES of data before we start.
-      @es.indices.delete_template(:name => "*")
-      # This can fail if there are no indexes, ignore failure.
-      @es.indices.delete(:index => "*") rescue nil
-      @es.index(
-        :index => 'logstash-update',
-        :type => doc_type,
-        :id => "123",
-        :body => { :message => 'Test', :counter => 1 }
-      )
-      @es.indices.refresh
+    it "should allow internal version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+      subject.register
     end
 
-    it "should fail without a document_id" do
-      subject = get_es_output
+    it "should not allow external version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
-    context "when update only" do
-      it "should not create new document" do
-        subject = get_es_output({ 'document_id' => "456" } )
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-      end
-
-      it "should update existing document" do
-        subject = get_es_output({ 'document_id' => "123" })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["message"]).to eq('updated message here')
-      end
-
-      # The es ruby client treats the data field differently. Make sure this doesn't
-      # raise an exception
-      it "should update an existing document that has a 'data' field" do
-        subject = get_es_output({ 'document_id' => "123" })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
-        expect(r["_source"]["data"]).to eq('updated message here')
-        expect(r["_source"]["message"]).to eq('foo')
-      end
-
-      it "should allow default (internal) version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99" })
-        subject.register
-      end
-
-      it "should allow internal version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
-        subject.register
-      end
-
-      it "should not allow external version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
-
-      it "should not allow external_gt version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
-
-      it "should not allow external_gte version" do
-        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
+    it "should not allow external_gt version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
 
+    it "should not allow external_gte version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
-    context "when update with upsert" do
-      it "should create new documents with provided upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('upsert message')
-      end
-
-      it "should create new documents with event/doc as upsert" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
-        expect(r["_source"]["message"]).to eq('sample message here')
-      end
-
-      it "should fail on documents with event/doc as upsert at external version" do
-        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
-        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-      end
+  end
+
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["message"]).to eq('upsert message')
+    end
+
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["message"]).to eq('sample message here')
+    end
+
+    it "should fail on documents with event/doc as upsert at external version" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,9 +1,5 @@
 require "logstash/devutils/rspec/spec_helper"
 
-unless defined?(LogStash::OSS)
-  LogStash::OSS = ENV['DISTRIBUTION'] != "default"
-end
-
 require "logstash/outputs/elasticsearch"
 
 module LogStash::Outputs::ElasticSearch::SpecHelper
@@ -11,4 +7,4 @@ end
 
 RSpec.configure do |config|
   config.include LogStash::Outputs::ElasticSearch::SpecHelper
-end
+end

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -31,18 +31,12 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
     # end
   end
 
-  @@logstash_oss = LogStash::OSS
-
   before(:each) do
-    change_constant :OSS, false, target: LogStash # assume non-OSS by default
-
     stub_plugin_register! if do_register
   end
 
   after(:each) do
     subject.close if do_register
-
-    change_constant :OSS, @@logstash_oss, target: LogStash
   end
 
   context "default configuration" do
@@ -115,14 +109,6 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       end
     end
 
-    it "uses data-streams on LS 8.0 (OSS)" do
-      change_constant :LOGSTASH_VERSION, '8.0.1' do
-        change_constant :OSS, true, target: LogStash do
-          expect( subject.data_stream_config? ).to be true
-        end
-      end
-    end
-
     context 'old ES' do
 
       let(:es_version) { '7.8.1' }

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -8,15 +8,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
   let(:es_node_versions) { [ "0.0.0" ] }
-  let(:oss) { true }
   let(:license_status) { 'active' }
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
 
   let(:manticore_double) { double("manticore a") }
   before(:each) do
-    stub_const('LogStash::OSS', oss)
-
     response_double = double("manticore response").as_null_object
     # Allow healtchecks
     allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
@@ -278,50 +275,46 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       allow(subject).to receive(:health_check_request)
     end
 
-    context "when using default logstash distribution" do
-      let(:oss) { false }
-
-      context "if ES doesn't return a valid license" do
-        let(:license_status) { nil }
+    context "if ES doesn't return a valid license" do
+      let(:license_status) { nil }
 
-        it "marks the url as dead" do
-          subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(0)
-        end
+      it "marks the url as dead" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(0)
+      end
 
-        it "logs a warning" do
-          expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
-          subject.update_initial_urls
-        end
+      it "logs a warning" do
+        expect(subject.license_checker).to receive(:warn_no_license).once.and_call_original
+        subject.update_initial_urls
       end
+    end
 
-      context "if ES returns a valid license" do
-        let(:license_status) { 'active' }
+    context "if ES returns a valid license" do
+      let(:license_status) { 'active' }
 
-        it "marks the url as active" do
-          subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
-        end
+      it "marks the url as active" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(1)
+      end
 
-        it "does not log a warning" do
-          expect(subject.license_checker).to_not receive(:warn_no_license)
-          expect(subject.license_checker).to_not receive(:warn_invalid_license)
-          subject.update_initial_urls
-        end
+      it "does not log a warning" do
+        expect(subject.license_checker).to_not receive(:warn_no_license)
+        expect(subject.license_checker).to_not receive(:warn_invalid_license)
+        subject.update_initial_urls
       end
+    end
 
-      context "if ES returns an invalid license" do
-        let(:license_status) { 'invalid' }
+    context "if ES returns an invalid license" do
+      let(:license_status) { 'invalid' }
 
-        it "marks the url as active" do
-          subject.update_initial_urls
-          expect(subject.alive_urls_count).to eq(1)
-        end
+      it "marks the url as active" do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(1)
+      end
 
-        it "logs a warning" do
-          expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
-          subject.update_initial_urls
-        end
+      it "logs a warning" do
+        expect(subject.license_checker).to receive(:warn_invalid_license).and_call_original
+        subject.update_initial_urls
       end
     end
   end

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -4,19 +4,19 @@ require "logstash/outputs/elasticsearch/template_manager"
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
   describe ".default_template_path" do
-    context "elasticsearch 1.x" do
-      it "chooses the 2x template" do
-        expect(described_class.default_template_path(1)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
+    context "elasticsearch 6.x" do
+      it "chooses the 6x template" do
+        expect(described_class.default_template_path(6)).to end_with("/templates/ecs-disabled/elasticsearch-6x.json")
       end
     end
-    context "elasticsearch 2.x" do
-      it "chooses the 2x template" do
-        expect(described_class.default_template_path(2)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
+    context "elasticsearch 7.x" do
+      it "chooses the 7x template" do
+        expect(described_class.default_template_path(7)).to end_with("/templates/ecs-disabled/elasticsearch-7x.json")
       end
     end
-    context "elasticsearch 5.x" do
-      it "chooses the 5x template" do
-        expect(described_class.default_template_path(5)).to end_with("/templates/ecs-disabled/elasticsearch-5x.json")
+    context "elasticsearch 8.x" do
+      it "chooses the 8x template" do
+        expect(described_class.default_template_path(8)).to end_with("/templates/ecs-disabled/elasticsearch-8x.json")
       end
     end
   end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -7,7 +7,7 @@ require "logstash/outputs/elasticsearch"
 describe LogStash::Outputs::ElasticSearch do
   subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
-  let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample }
+  let(:maximum_seen_major_version) { [6,7,8].sample }
 
   let(:do_register) { true }
 
@@ -79,13 +79,6 @@ describe LogStash::Outputs::ElasticSearch do
             expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc")
           end
         end
-
-        context "for < 6.0 elasticsearch clusters" do
-          let(:maximum_seen_major_version) { 5 }
-          it "should get the type from the event" do
-            expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo")
-          end
-        end
       end
 
       context "with 'document type set'" do
@@ -941,6 +934,7 @@ describe LogStash::Outputs::ElasticSearch do
       allow(subject).to receive(:last_es_version).and_return es_version
       # make successful_connection? return true:
       allow(subject).to receive(:maximum_seen_major_version).and_return Integer(es_version.split('.').first)
+      allow(subject).to receive(:alive_urls_count).and_return Integer(1)
       allow(subject).to receive(:stop_after_successful_connection_thread)
     end
 

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -12,6 +12,7 @@ describe "whitelisting error types in expected behavior" do
   before :each do
     allow(subject.logger).to receive(:warn)
     allow(subject).to receive(:maximum_seen_major_version).and_return(0)
+    allow(subject).to receive(:alive_urls_count).and_return(1)
     allow(subject).to receive(:finish_register)
 
     subject.register

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.0.1
+  version: 11.0.5
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2021-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.7.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.4
+        version: 0.7.1
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -179,26 +179,21 @@ files:
 - lib/logstash/outputs/elasticsearch/ilm.rb
 - lib/logstash/outputs/elasticsearch/license_checker.rb
 - lib/logstash/outputs/elasticsearch/template_manager.rb
-- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json
-- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
 - lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
 - lib/logstash/plugin_mixins/elasticsearch/common.rb
 - lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
-- spec/fixtures/_nodes/2x_1x.json
-- spec/fixtures/_nodes/5x_6x.json
+- spec/fixtures/_nodes/6x.json
 - spec/fixtures/_nodes/7x.json
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
-- spec/fixtures/scripts/groovy/scripted_update.groovy
-- spec/fixtures/scripts/groovy/scripted_update_nested.groovy
-- spec/fixtures/scripts/groovy/scripted_upsert.groovy
 - spec/fixtures/scripts/painless/scripted_update.painless
 - spec/fixtures/scripts/painless/scripted_update_nested.painless
 - spec/fixtures/scripts/painless/scripted_upsert.painless
@@ -213,7 +208,6 @@ files:
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/data_stream_spec.rb
 - spec/integration/outputs/delete_spec.rb
-- spec/integration/outputs/groovy_update_spec.rb
 - spec/integration/outputs/ilm_spec.rb
 - spec/integration/outputs/index_spec.rb
 - spec/integration/outputs/index_version_spec.rb
@@ -225,7 +219,6 @@ files:
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
-- spec/integration/outputs/templates_5x_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
 - spec/spec_helper.rb
@@ -266,21 +259,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Stores logs in Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
-- spec/fixtures/_nodes/2x_1x.json
-- spec/fixtures/_nodes/5x_6x.json
+- spec/fixtures/_nodes/6x.json
 - spec/fixtures/_nodes/7x.json
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
-- spec/fixtures/scripts/groovy/scripted_update.groovy
-- spec/fixtures/scripts/groovy/scripted_update_nested.groovy
-- spec/fixtures/scripts/groovy/scripted_upsert.groovy
 - spec/fixtures/scripts/painless/scripted_update.painless
 - spec/fixtures/scripts/painless/scripted_update_nested.painless
 - spec/fixtures/scripts/painless/scripted_upsert.painless
@@ -295,7 +283,6 @@ test_files:
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/data_stream_spec.rb
 - spec/integration/outputs/delete_spec.rb
-- spec/integration/outputs/groovy_update_spec.rb
 - spec/integration/outputs/ilm_spec.rb
 - spec/integration/outputs/index_spec.rb
 - spec/integration/outputs/index_version_spec.rb
@@ -307,7 +294,6 @@ test_files:
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
-- spec/integration/outputs/templates_5x_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
 - spec/spec_helper.rb