logstash-output-elasticsearch 11.22.16-java → 12.0.0-java

data/spec/integration/outputs/unsupported_actions_spec.rb

@@ -27,21 +27,16 @@ describe "Unsupported actions testing...", :integration => true do
     @es.indices.delete(:index => "*") rescue nil
     # index single doc for update purpose
     @es.index(
-      {
-        :index => INDEX,
-        :id => '2',
-        :body => { :message => 'Test to doc indexing', :counter => 1 },
-        :refresh => true
-      }
+      :index => INDEX,
+      :type => doc_type,
+      :id => "2",
+      :body => { :message => 'Test to doc indexing', :counter => 1 }
     )
-
     @es.index(
-      {
-        :index => INDEX,
-        :id => '3',
-        :body => { :message => 'Test to doc deletion', :counter => 2 },
-        :refresh => true
-      }
+      :index => INDEX,
+      :type => doc_type,
+      :id => "3",
+      :body => { :message => 'Test to doc deletion', :counter => 2 }
     )
     @es.indices.refresh
   end
@@ -68,12 +63,12 @@ describe "Unsupported actions testing...", :integration => true do
       rejected_events = events.select { |event| !index_or_update.call(event) }
 
       indexed_events.each do |event|
-        response = @es.get(:index => INDEX, :id => event.get("doc_id"), :refresh => true)
+        response = @es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)
         expect(response['_source']['message']).to eq(event.get("message"))
       end
 
       rejected_events.each do |event|
-        expect {@es.get(:index => INDEX, :id => event.get("doc_id"), :refresh => true)}.to raise_error(get_expected_error_class)
+        expect {@es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
       end
     end
   end

data/spec/integration/outputs/update_spec.rb

@@ -22,12 +22,10 @@ describe "Update actions without scripts", :integration => true do
     # This can fail if there are no indexes, ignore failure.
     @es.indices.delete(:index => "*") rescue nil
     @es.index(
-      {
-        :index => 'logstash-update',
-        :id => '123',
-        :body => { :message => 'Test', :counter => 1 },
-        :refresh => true
-      }
+      :index => 'logstash-update',
+      :type => doc_type,
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
     )
     @es.indices.refresh
   end
@@ -42,14 +40,14 @@ describe "Update actions without scripts", :integration => true do
       subject = get_es_output({ 'document_id' => "456" } )
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      expect {@es.get(:index => 'logstash-update', :id => '456', :refresh => true)}.to raise_error(get_expected_error_class)
+      expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
     end
 
     it "should update existing document" do
       subject = get_es_output({ 'document_id' => "123" })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
-      r = @es.get(:index => 'logstash-update', :id => '123', :refresh => true)
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
       expect(r["_source"]["message"]).to eq('updated message here')
     end
 
@@ -59,7 +57,7 @@ describe "Update actions without scripts", :integration => true do
       subject = get_es_output({ 'document_id' => "123" })
       subject.register
       subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
-      r = @es.get(:index => 'logstash-update', :id => '123', :refresh => true)
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
       expect(r["_source"]["data"]).to eq('updated message here')
       expect(r["_source"]["message"]).to eq('foo')
     end
@@ -96,7 +94,7 @@ describe "Update actions without scripts", :integration => true do
       subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :id => '456', :refresh => true)
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
       expect(r["_source"]["message"]).to eq('upsert message')
     end
 
@@ -104,7 +102,7 @@ describe "Update actions without scripts", :integration => true do
       subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :id => '456', :refresh => true)
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
       expect(r["_source"]["message"]).to eq('sample message here')
     end
 

data/spec/spec_helper.rb

@@ -8,11 +8,3 @@ end
 RSpec.configure do |config|
   config.include LogStash::Outputs::ElasticSearch::SpecHelper
 end
-
-# remove once plugin starts consuming elasticsearch-ruby v8 client
-def elastic_ruby_v8_client_available?
-  Elasticsearch::Transport
-  false
-rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
-  true
-end

data/spec/support/elasticsearch/api/actions/get_alias.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # Retrieve the list of index lifecycle management policies
+      def get_alias(arguments={})
+        method = HTTP_GET
+        path   = Utils.__pathify '_alias', Utils.__escape(arguments[:name])
+        params = {}
+        perform_request(method, path, params, nil).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/put_alias.rb

@@ -0,0 +1,24 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # @option arguments [String] :name The name of the alias (*Required*)
+      # @option arguments [Hash]   :The alias definition(*Required*)
+
+      def put_alias(arguments={})
+        raise ArgumentError, "Required argument 'name' missing" unless arguments[:name]
+        raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+        method = HTTP_PUT
+        path   = Utils.__pathify Utils.__escape(arguments[:name])
+
+        params = Utils.__validate_and_extract_params arguments
+        body   = arguments[:body]
+        perform_request(method, path, params, body.to_json).body
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -270,83 +270,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
        end
      end
-    context "the 'user-agent' header" do
-      let(:pool) { double("pool") }
-      let(:compression_level) { 6 }
-      let(:base_options) { super().merge( :client_settings => {:compression_level => compression_level}) }
-      let(:actions) { [
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message_1"=> message_1}],
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message_2"=> message_2}],
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message_3"=> message_3}],
-      ]}
-      let(:message_1) { "hello" }
-      let(:message_2_size) { 1_000 }
-      let(:message_2) { SecureRandom.alphanumeric(message_2_size / 2 ) * 2 }
-      let(:message_3_size) { 1_000 }
-      let(:message_3) { "m" * message_3_size }
-      let(:messages_size) { message_1.size + message_2.size + message_3.size }
-      let(:action_overhead) { 42 + 16 + 2 } # header plus doc key size plus new line overhead per action
-
-      let(:response) do
-        response = double("response")
-        allow(response).to receive(:code).and_return(response)
-        allow(response).to receive(:body).and_return({"errors" => false}.to_json)
-        response
-      end
-
-      before(:each) do
-        subject.instance_variable_set("@pool", pool)
-      end
-
-      it "carries bulk request's uncompressed size" do
-        expect(pool).to receive(:post) do |path, params, body|
-          headers = params.fetch(:headers, {})
-          expect(headers["X-Elastic-Event-Count"]).to eq("3")
-          expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
-        end.and_return(response)
-
-        subject.send(:bulk, actions)
-      end
-      context "without compression" do
-        let(:compression_level) { 0 }
-        it "carries bulk request's uncompressed size" do
-          expect(pool).to receive(:post) do |path, params, body|
-            headers = params.fetch(:headers, {})
-            expect(headers["X-Elastic-Event-Count"]).to eq("3")
-            expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (messages_size + (action_overhead * 3)).to_s
-          end.and_return(response)
-          subject.send(:bulk, actions)
-        end
-      end
-
-      context "with compressed messages over 20MB" do
-        let(:message_2_size) { 21_000_000 }
-        it "carries bulk request's uncompressed size" do
-          # only the first, tiny, message is sent first
-          expect(pool).to receive(:post) do |path, params, body|
-            headers = params.fetch(:headers, {})
-            expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_1.size + action_overhead).to_s
-            expect(headers["X-Elastic-Event-Count"]).to eq("1")
-          end.and_return(response)
-
-          # huge message_2 is sent afterwards alone
-          expect(pool).to receive(:post) do |path, params, body|
-            headers = params.fetch(:headers, {})
-            expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_2.size + action_overhead).to_s
-            expect(headers["X-Elastic-Event-Count"]).to eq("1")
-          end.and_return(response)
-
-          # finally medium message_3 is sent alone as well
-          expect(pool).to receive(:post) do |path, params, body|
-            headers = params.fetch(:headers, {})
-            expect(headers["X-Elastic-Uncompressed-Request-Length"]).to eq (message_3.size + action_overhead).to_s
-            expect(headers["X-Elastic-Event-Count"]).to eq("1")
-          end.and_return(response)
-
-          subject.send(:bulk, actions)
-        end
-      end
-    end
   end
 
   describe "sniffing" do

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -186,75 +186,4 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
     end
   end
-
-  describe ".strip_serverless_incompatible_settings" do
-    context "with composable template (template.settings)" do
-      let(:template) do
-        {
-          "template" => {
-            "settings" => {
-              "index.refresh_interval" => "5s",
-              "number_of_shards" => 1,
-              "number_of_replicas" => 0
-            }
-          }
-        }
-      end
-
-      it "removes number_of_shards and number_of_replicas" do
-        described_class.strip_serverless_incompatible_settings(template)
-        expect(template["template"]["settings"]).not_to have_key("number_of_shards")
-        expect(template["template"]["settings"]).not_to have_key("number_of_replicas")
-      end
-
-      it "preserves compatible settings" do
-        described_class.strip_serverless_incompatible_settings(template)
-        expect(template["template"]["settings"]["index.refresh_interval"]).to eq("5s")
-      end
-    end
-
-    context "with index-prefixed setting keys" do
-      let(:template) do
-        {
-          "template" => {
-            "settings" => {
-              "index.number_of_shards" => 1,
-              "index.number_of_replicas" => 0
-            }
-          }
-        }
-      end
-
-      it "removes index-prefixed variants" do
-        described_class.strip_serverless_incompatible_settings(template)
-        expect(template["template"]["settings"]).not_to have_key("index.number_of_shards")
-        expect(template["template"]["settings"]).not_to have_key("index.number_of_replicas")
-      end
-    end
-
-    context "with legacy template (top-level settings)" do
-      let(:template) do
-        {
-          "settings" => {
-            "index.refresh_interval" => "5s",
-            "number_of_shards" => 1
-          }
-        }
-      end
-
-      it "removes incompatible settings from top-level settings" do
-        described_class.strip_serverless_incompatible_settings(template)
-        expect(template["settings"]).not_to have_key("number_of_shards")
-        expect(template["settings"]["index.refresh_interval"]).to eq("5s")
-      end
-    end
-
-    context "with no settings" do
-      let(:template) { { "mappings" => {} } }
-
-      it "does not raise" do
-        expect { described_class.strip_serverless_incompatible_settings(template) }.not_to raise_error
-      end
-    end
-  end
 end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -777,7 +777,7 @@ describe LogStash::Outputs::ElasticSearch do
       end
 
       before(:each) do
-        allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array), instance_of(Hash)) do |stream, actions, headers|
+        allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
           expect( stream.string ).to include '"foo":"bar1"'
           expect( stream.string ).to include '"foo":"bar2"'
         end.and_return(bulk_response, {"errors"=>false}) # let's make it go away (second call) to not retry indefinitely
@@ -908,9 +908,6 @@ describe LogStash::Outputs::ElasticSearch do
 
     before(:each) do
       allow(elasticsearch_output_instance.client).to receive(:logger).and_return(logger_stub)
-      # Stub the output plugin's own @logger to prevent submit()/retrying_submit()
-      # from logging the full 15MB action payload on each retry, which floods stdout.
-      elasticsearch_output_instance.instance_variable_set(:@logger, logger_stub)
 
       allow(elasticsearch_output_instance.client).to receive(:bulk).and_call_original
 
@@ -918,12 +915,7 @@ describe LogStash::Outputs::ElasticSearch do
       allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body|
         if body.length > max_bytes
           max_bytes *= 2 # ensure a successful retry
-          raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
-            413,
-            "test-url",
-            body,
-            ""
-          )
+          double("Response", :code => 413, :body => "")
         else
           double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}')
         end
@@ -1133,81 +1125,6 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
-  describe "SSL deprecated settings" do
-    let(:base_options) { {"ssl" => "true"} }
-
-    context "with client certificate" do
-      let(:do_register) { true }
-      let(:cacert) { Stud::Temporary.file.path }
-      let(:options) { base_options.merge(
-        "cacert" => cacert,
-        "ssl_certificate_verification" => false
-      ) }
-
-      after :each do
-        File.delete(cacert)
-      end
-
-      it "should map new configs into params" do
-        expect(subject.params).to match hash_including(
-                                          "ssl_enabled" => true,
-                                          "ssl_verification_mode" => "none",
-                                          "ssl_certificate_authorities" => [cacert]
-                                        )
-      end
-
-      it "should set new configs variables" do
-        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
-        expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("none")
-        expect(subject.instance_variable_get(:@ssl_certificate_authorities)).to eql([cacert])
-      end
-    end
-
-    context "with java stores" do
-      let(:do_register) { true }
-      let(:keystore) { Stud::Temporary.file.path }
-      let(:truststore) { Stud::Temporary.file.path }
-      let(:options) { base_options.merge(
-        "keystore" => keystore,
-        "keystore_password" => "keystore",
-        "truststore" => truststore,
-        "truststore_password" => "truststore",
-        "ssl_certificate_verification" => true
-      ) }
-
-      let(:spy_http_client_builder!) do
-        allow(described_class::HttpClientBuilder).to receive(:build).with(any_args).and_call_original
-        allow(described_class::HttpClientBuilder).to receive(:setup_ssl).with(any_args).and_return({})
-      end
-
-      after :each do
-        File.delete(keystore)
-        File.delete(truststore)
-      end
-
-      it "should map new configs into params" do
-        expect(subject.params).to match hash_including(
-                                          "ssl_enabled" => true,
-                                          "ssl_keystore_path" => keystore,
-                                          "ssl_truststore_path" => truststore,
-                                          "ssl_verification_mode" => "full"
-                                        )
-
-        expect(subject.params["ssl_keystore_password"].value).to eql("keystore")
-        expect(subject.params["ssl_truststore_password"].value).to eql("truststore")
-      end
-
-      it "should set new configs variables" do
-        expect(subject.instance_variable_get(:@ssl_enabled)).to eql(true)
-        expect(subject.instance_variable_get(:@ssl_keystore_path)).to eql(keystore)
-        expect(subject.instance_variable_get(:@ssl_keystore_password).value).to eql("keystore")
-        expect(subject.instance_variable_get(:@ssl_truststore_path)).to eql(truststore)
-        expect(subject.instance_variable_get(:@ssl_truststore_password).value).to eql("truststore")
-        expect(subject.instance_variable_get(:@ssl_verification_mode)).to eql("full")
-      end
-    end
-  end
-
   describe "retry_on_conflict" do
     let(:num_retries) { 123 }
     let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") }
@@ -1560,7 +1477,7 @@ describe LogStash::Outputs::ElasticSearch do
             expect( event ).to be_a LogStash::Event
             expect( event ).to be events.first
             expect( reason ).to start_with "Could not index event to Elasticsearch. status: #{error_code}, action: [\"index\""
-            expect( reason ).to match /_id(?::| ?=>) ?"bar".*"foo" ?=> ?"bar".*response:.*"reason" ?=> ?"TEST"/
+            expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
 
             method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
           end.once

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -195,3 +195,25 @@ describe "SSL options" do
   end
 end
 
+# Move outside the SSL options describe block that has the after hook
+describe "SSL obsolete settings" do
+  let(:base_settings) { { "hosts" => "localhost", "pool_max" => 1, "pool_max_per_route" => 1 } }
+  [
+    {name: 'ssl', replacement: 'ssl_enabled'},
+    {name: 'ssl_certificate_verification', replacement: 'ssl_verification_mode'},
+    {name: 'cacert', replacement: 'ssl_certificate_authorities'},
+    {name: 'truststore', replacement: 'ssl_truststore_path'},
+    {name: 'keystore', replacement: 'ssl_keystore_path'},
+    {name: 'truststore_password', replacement: 'ssl_truststore_password'},
+    {name: 'keystore_password', replacement: 'ssl_keystore_password'}
+  ].each do |obsolete_setting|
+    context "with option #{obsolete_setting[:name]}" do
+      let(:settings) { base_settings.merge(obsolete_setting[:name] => "value") }
+
+      it "emits an error about the setting being obsolete" do
+        error_text = /The setting `#{obsolete_setting[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. (Use|Set) '#{obsolete_setting[:replacement]}' instead/i
+        expect { LogStash::Outputs::ElasticSearch.new(settings) }.to raise_error LogStash::ConfigurationError, error_text
+      end
+    end
+  end
+end

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -4,6 +4,7 @@ require_relative "../../../spec/es_spec_helper"
 describe "whitelisting error types in expected behavior" do
   let(:template) { '{"template" : "not important, will be updated by :index"}' }
   let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+  let(:action1) { ["index", {:_id=>1, :routing=>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1] }
   let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } }
 
   subject { LogStash::Outputs::ElasticSearch.new(settings) }