logstash-input-elasticsearch 4.23.1 → 5.0.0

data/spec/inputs/elasticsearch_spec.rb

@@ -21,13 +21,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   let(:es_version) { "7.5.0" }
   let(:cluster_info) { {"version" => {"number" => es_version, "build_flavor" => build_flavor}, "tagline" => "You Know, for Search"} }
 
-  def elastic_ruby_v8_client_available?
-    Elasticsearch::Transport
-    false
-  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
-    true
-  end
-
   before(:each) do
     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
     allow_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(cluster_info)
@@ -65,6 +58,19 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
     end
 
+    describe 'handling obsolete settings' do
+      [{:name => 'ssl', :replacement => 'ssl_enabled', :sample_value => true},
+       {:name => 'ca_file', :replacement => 'ssl_certificate_authorities', :sample_value => 'spec/fixtures/test_certs/ca.crt'},
+       {:name => 'ssl_certificate_verification', :replacement => 'ssl_verification_mode', :sample_value => false }].each do | obsolete_setting|
+        context "with obsolete #{obsolete_setting[:name]}" do
+          let (:config) { {obsolete_setting[:name] => obsolete_setting[:sample_value]} }
+          it "should raise a config error with the appropriate message" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /The setting `#{obsolete_setting[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. Set '#{obsolete_setting[:replacement]}' instead/i
+          end
+        end
+      end
+    end
+
     context "against not authentic Elasticsearch" do
       before(:each) do
          Elasticsearch::Client.send(:define_method, :ping) { raise Elasticsearch::UnsupportedProductError.new("Fake error") } # define error ping method
@@ -86,11 +92,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
         before do
           allow(Elasticsearch::Client).to receive(:new).and_return(es_client)
-          if elastic_ruby_v8_client_available?
-            allow(es_client).to receive(:info).and_raise(Elastic::Transport::Transport::Errors::BadRequest.new)
-          else
-            allow(es_client).to receive(:info).and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest.new)
-          end
+          allow(es_client).to receive(:info).and_raise(
+            Elasticsearch::Transport::Transport::Errors::BadRequest.new
+          )
         end
 
         it "raises an exception" do
@@ -662,28 +666,11 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         context 'if the `docinfo_target` exist but is not of type hash' do
           let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
           let(:do_register) { false }
-          let(:mock_queue) { double('Queue', :<< => nil) }
-          let(:hit) { response.dig('hits', 'hits').first }
-
-          it 'emits a tagged event with JSON-serialized event in [event][original]' do
-            allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
 
+          it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
+            expect(client).not_to receive(:clear_scroll)
             plugin.register
-            plugin.run(mock_queue)
-
-            expect(mock_queue).to have_received(:<<) do |event|
-              expect(event).to be_a_kind_of LogStash::Event
-
-              expect(event.get('tags')).to include("_elasticsearch_input_failure")
-              expect(event.get('[event][original]')).to be_a_kind_of String
-              expect(JSON.load(event.get('[event][original]'))).to eq hit
-            end
-
-            expect(plugin.logger)
-              .to have_received(:warn).with(
-                a_string_including("Event creation error, original data now in [event][original] field"),
-                a_hash_including(:message => a_string_including('unable to merge docinfo fields into docinfo_target=`metadata_with_string`'),
-                                 :data    => a_string_including('"_id":"C5b2xLQwTZa76jBmHIbwHQ"')))
+            expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
           end
 
         end
@@ -740,13 +727,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:client)
-        target_field = :@seeds
-        begin
-          Elasticsearch::Transport::Client
-        rescue
-          target_field = :@hosts
-        end
-        expect( client.transport.instance_variable_get(target_field) ).to eql [{
+
+        expect( client.transport.instance_variable_get(:@seeds) ).to eql [{
                                                                               :scheme => "https",
                                                                               :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
                                                                               :port => 9243,
@@ -810,50 +792,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
 
       context "with ssl" do
-        let(:api_key_value) { nil }
-        let(:config) { super().merge("ssl_enabled" => true, 'api_key' => LogStash::Util::Password.new(api_key_value)) }
-        let(:encoded_api_key) { Base64.strict_encode64('foo:bar') }
-
-        shared_examples "a plugin that sets the ApiKey authorization header" do
-          it "correctly sets the Authorization header" do
-            plugin.register
-            client = plugin.send(:client)
-            auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
-
-            expect(auth_header).to eql("ApiKey #{encoded_api_key}")
-          end
-        end
-
-        context "with a non-encoded API key" do
-          let(:api_key_value) { "foo:bar" }
-          it_behaves_like "a plugin that sets the ApiKey authorization header"
-        end
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl_enabled" => true }) }
 
-        context "with an encoded API key" do
-          let(:api_key_value) { encoded_api_key }
-          it_behaves_like "a plugin that sets the ApiKey authorization header"
-        end
-
-        context "with an Elastic Cloud API key (essu_ prefix)" do
-          # The suffix is intentionally not canonical base64: a Cloud key is opaque
-          # and must be forwarded verbatim regardless of its payload encoding.
-          let(:api_key_value) { "essu_VFZGblZreFhTekJ4ZDB4M2NHUnZRMEU2YzNWd1pYSnpaV055WlhRPQ==AAAAAAAA" }
-
-          it "sets the Authorization header verbatim without re-encoding" do
-            plugin.register
-            client = plugin.send(:client)
-            auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
-
-            expect(auth_header).to eql("ApiKey #{api_key_value}")
-          end
-        end
-
-        context "with an unrecognized api_key format" do
-          let(:api_key_value) { "not-a-valid-key" }
+        it "should set authorization" do
+          plugin.register
+          client = plugin.send(:client)
+          auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
-          it "fails registration with a configuration error" do
-            expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Invalid api_key format/)
-          end
+          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
         end
 
         context 'user also set' do
@@ -1188,7 +1134,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
     context "when there's an exception" do
       before(:each) do
-        allow(client).to receive(:search).and_raise RuntimeError.new("test exception")
+        allow(client).to receive(:search).and_raise RuntimeError
       end
       it 'produces no events' do
         plugin.run queue
@@ -1302,220 +1248,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
   end
 
-  context '#push_hit' do
-    let(:config) do
-      {
-        'docinfo' => true, # include ids
-        'docinfo_target' => '[@metadata][docinfo]'
-      }
-    end
-
-    let(:hit) do
-      JSON.load(<<~EOJSON)
-        {
-          "_index" : "test_bulk_index_2",
-          "_type" : "_doc",
-          "_id" : "sHe6A3wBesqF7ydicQvG",
-          "_score" : 1.0,
-          "_source" : {
-            "@timestamp" : "2021-09-20T15:02:02.557Z",
-            "message" : "ping",
-            "@version" : "17",
-            "sequence" : 7,
-            "host" : {
-              "name" : "maybe.local",
-              "ip" : "127.0.0.1"
-            }
-          }
-        }
-      EOJSON
-    end
-
-    let(:mock_queue) { double('queue', :<< => nil) }
-
-    before(:each) do
-      plugin.send(:setup_cursor_tracker)
-    end
-
-    it 'pushes a generated event to the queue' do
-      plugin.send(:push_hit, hit, mock_queue)
-      expect(mock_queue).to have_received(:<<) do |event|
-        expect(event).to be_a_kind_of LogStash::Event
-
-        # fields overriding defaults
-        expect(event.timestamp.to_s).to eq("2021-09-20T15:02:02.557Z")
-        expect(event.get('@version')).to eq("17")
-
-        # structure from hit's _source
-        expect(event.get('message')).to eq("ping")
-        expect(event.get('sequence')).to eq(7)
-        expect(event.get('[host][name]')).to eq("maybe.local")
-        expect(event.get('[host][ip]')).to eq("127.0.0.1")
-
-        # docinfo fields
-        expect(event.get('[@metadata][docinfo][_index]')).to eq("test_bulk_index_2")
-        expect(event.get('[@metadata][docinfo][_type]')).to eq("_doc")
-        expect(event.get('[@metadata][docinfo][_id]')).to eq("sHe6A3wBesqF7ydicQvG")
-      end
-    end
-
-    context 'when event creation fails' do
-      before(:each) do
-        allow(plugin).to receive(:logger).and_return(double('Logger').as_null_object)
-
-        allow(plugin.event_factory).to receive(:new_event).and_call_original
-        allow(plugin.event_factory).to receive(:new_event).with(a_hash_including hit['_source']).and_raise(RuntimeError, 'intentional')
-      end
-
-      it 'pushes a tagged event containing a JSON-encoded hit in [event][original]' do
-        plugin.send(:push_hit, hit, mock_queue)
-
-        expect(mock_queue).to have_received(:<<) do |event|
-          expect(event).to be_a_kind_of LogStash::Event
-
-          expect(event.get('tags')).to include("_elasticsearch_input_failure")
-          expect(event.get('[event][original]')).to be_a_kind_of String
-          expect(JSON.load(event.get('[event][original]'))).to eq hit
-        end
-
-        expect(plugin.logger)
-          .to have_received(:warn).with(
-            a_string_including("Event creation error, original data now in [event][original] field"),
-            a_hash_including(:message => a_string_including('intentional'),
-                             :data    => a_string_including('"_id":"sHe6A3wBesqF7ydicQvG"')))
-
-      end
-    end
-  end
-
   # @note can be removed once we depends on elasticsearch gem >= 6.x
   def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
-  describe "#ESQL" do
-    let(:config) do
-      {
-        "query" => "FROM test-index | STATS count() BY field",
-        "query_type" => "esql",
-        "retries" => 3
-      }
-    end
-    let(:es_version) { LogStash::Inputs::Elasticsearch::ES_ESQL_SUPPORT_VERSION }
-    let(:ls_version) { LogStash::Inputs::Elasticsearch::LS_ESQL_SUPPORT_VERSION }
-
-    before(:each) do
-      stub_const("LOGSTASH_VERSION", ls_version)
-    end
-
-    describe "#initialize" do
-      it "sets up the ESQL client with correct parameters" do
-        expect(plugin.instance_variable_get(:@query_type)).to eq(config["query_type"])
-        expect(plugin.instance_variable_get(:@query)).to eq(config["query"])
-        expect(plugin.instance_variable_get(:@retries)).to eq(config["retries"])
-      end
-    end
-
-    describe "#register" do
-      before(:each) do
-        Elasticsearch::Client.send(:define_method, :ping) { }
-        allow_any_instance_of(Elasticsearch::Client).to receive(:info).and_return(cluster_info)
-      end
-      it "creates ES|QL executor" do
-        plugin.register
-        expect(plugin.instance_variable_get(:@query_executor)).to be_an_instance_of(LogStash::Inputs::Elasticsearch::Esql)
-      end
-    end
-
-    describe "#validation" do
-
-      describe "LS version" do
-        context "when compatible" do
-
-          it "does not raise an error" do
-            expect { plugin.send(:validate_ls_version_for_esql_support!) }.not_to raise_error
-          end
-        end
-
-        context "when incompatible" do
-          before(:each) do
-            stub_const("LOGSTASH_VERSION", "8.10.0")
-          end
-
-          it "raises a runtime error" do
-            expect { plugin.send(:validate_ls_version_for_esql_support!) }
-              .to raise_error(RuntimeError, /Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{ls_version}/)
-          end
-        end
-      end
-
-      describe "ES version" do
-        before(:each) do
-          allow(plugin).to receive(:es_version).and_return("8.10.5")
-        end
-
-        context "when incompatible" do
-          it "raises a runtime error" do
-            expect { plugin.send(:validate_es_for_esql_support!) }
-              .to raise_error(RuntimeError, /Connected Elasticsearch 8.10.5 version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{es_version} version./)
-          end
-        end
-      end
-
-      context "ES|QL query and DSL params used together" do
-        let(:config) {
-          super().merge({
-            "index" => "my-index",
-            "size" => 1,
-            "slices" => 1,
-            "search_api" => "auto",
-            "docinfo" => true,
-            "docinfo_target" => "[@metadata][docinfo]",
-            "docinfo_fields" => ["_index"],
-            "response_type" => "hits",
-            "tracking_field" => "[@metadata][tracking]"
-          })}
-
-        it "raises a config error" do
-          mixed_fields = %w[index size slices docinfo_fields response_type tracking_field]
-          expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Configured #{mixed_fields} params are not allowed while using ES|QL query/)
-        end
-      end
-
-      describe "ES|QL query"  do
-        context "when query is valid" do
-          it "does not raise an error" do
-            expect { plugin.send(:validate_esql_query!) }.not_to raise_error
-          end
-        end
-
-        context "when query is empty" do
-          let(:config) do
-            {
-              "query" => " "
-            }
-          end
-
-          it "raises a configuration error" do
-            expect { plugin.send(:validate_esql_query!) }
-              .to raise_error(LogStash::ConfigurationError, /`query` cannot be empty/)
-          end
-        end
-
-        context "when query doesn't align with ES syntax" do
-          let(:config) do
-            {
-              "query" => "RANDOM query"
-            }
-          end
-
-          it "raises a configuration error" do
-            source_commands = %w[FROM ROW SHOW]
-            expect { plugin.send(:validate_esql_query!) }
-              .to raise_error(LogStash::ConfigurationError, "`query` needs to start with any of #{source_commands}")
-          end
-        end
-      end
-    end
-  end
 end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -4,7 +4,7 @@ require "logstash/plugin"
 require "logstash/inputs/elasticsearch"
 require_relative "../../../spec/es_helper"
 
-describe LogStash::Inputs::Elasticsearch do
+describe LogStash::Inputs::Elasticsearch, :integration => true do
 
   SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
 
@@ -76,14 +76,6 @@ describe LogStash::Inputs::Elasticsearch do
     shared_examples 'secured_elasticsearch' do
       it_behaves_like 'an elasticsearch index plugin'
 
-      let(:unauth_exception_class) do
-        begin
-          Elasticsearch::Transport::Transport::Errors::Unauthorized
-        rescue
-          Elastic::Transport::Transport::Errors::Unauthorized
-        end
-      end
-
       context "incorrect auth credentials" do
 
         let(:config) do
@@ -93,7 +85,7 @@ describe LogStash::Inputs::Elasticsearch do
         let(:queue) { [] }
 
         it "fails to run the plugin" do
-          expect { plugin.register }.to raise_error unauth_exception_class
+          expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
         end
       end
     end