logstash-output-opensearch 1.0.0-java

data/spec/unit/outputs/opensearch/http_client/pool_spec.rb

@@ -0,0 +1,306 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/opensearch/http_client"
+require 'cabin'
+
+describe LogStash::Outputs::OpenSearch::HttpClient::Pool do
+  let(:logger) { Cabin::Channel.get }
+  let(:adapter) { LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
+  let(:node_versions) { [ "7.0.0" ] }
+  let(:get_distribution) { "oss" }
+
+  subject { described_class.new(logger, adapter, initial_urls, options) }
+
+  let(:manticore_double) { double("manticore a") }
+  before(:each) do
+    response_double = double("manticore response").as_null_object
+    # Allow healtchecks
+    allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:close)
+
+    allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
+
+    allow(subject).to receive(:get_version).with(any_args).and_return(*node_versions)
+    allow(subject.distribution_checker).to receive(:get_distribution).and_return(get_distribution)
+  end
+
+  after do
+    subject.close
+  end
+
+  describe "initialization" do
+    it "should be successful" do
+      expect { subject }.not_to raise_error
+      subject.start
+    end
+  end
+
+  describe "the resurrectionist" do
+    before(:each) { subject.start }
+    it "should start the resurrectionist when created" do
+      expect(subject.resurrectionist_alive?).to eql(true)
+    end
+
+    it "should attempt to resurrect connections after the ressurrect delay" do
+      expect(subject).to receive(:healthcheck!).once
+      sleep(subject.resurrect_delay + 1)
+    end
+
+    describe "healthcheck url handling" do
+      let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+
+      context "and not setting healthcheck_path" do
+        it "performs the healthcheck to the root" do
+          expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
+            expect(method).to eq(:head)
+            expect(url.path).to be_empty
+            expect(req_path).to eq("/")
+          end
+          subject.healthcheck!
+        end
+      end
+
+      context "and setting healthcheck_path" do
+        let(:healthcheck_path) { "/my/health" }
+        let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
+        it "performs the healthcheck to the healthcheck_path" do
+          expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
+            expect(method).to eq(:head)
+            expect(url.path).to be_empty
+            expect(req_path).to eq(healthcheck_path)
+          end
+          subject.healthcheck!
+        end
+      end
+    end
+  end
+
+  describe 'resolving the address from OpenSearch node info' do
+    let(:host) { "unit-test-node"}
+    let(:ip_address) { "192.168.1.0"}
+    let(:port) { 9200 }
+
+    context 'with host and ip address' do
+      let(:publish_address) { "#{host}/#{ip_address}:#{port}"}
+      it 'should correctly extract the host' do
+        expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
+      end
+    end
+    context 'with ip address' do
+      let(:publish_address) { "#{ip_address}:#{port}"}
+      it 'should correctly extract the ip address' do
+        expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
+      end
+    end
+  end
+
+  describe "the sniffer" do
+    before(:each) { subject.start }
+    it "should not start the sniffer by default" do
+      expect(subject.sniffer_alive?).to eql(nil)
+    end
+
+    context "when enabled" do
+      let(:options) { super().merge(:sniffing => true)}
+
+      it "should start the sniffer" do
+        expect(subject.sniffer_alive?).to eql(true)
+      end
+    end
+  end
+
+  describe "closing" do
+    before do
+      subject.start
+      # Simulate a single in use connection on the first check of this
+      allow(adapter).to receive(:close).and_call_original
+      allow(subject).to receive(:wait_for_in_use_connections).and_call_original
+      allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
+      allow(subject).to receive(:start)
+      subject.close
+    end
+
+    it "should close the adapter" do
+      expect(adapter).to have_received(:close)
+    end
+
+    it "should stop the resurrectionist" do
+      expect(subject.resurrectionist_alive?).to eql(false)
+    end
+
+    it "should stop the sniffer" do
+      # If no sniffer (the default) returns nil
+      expect(subject.sniffer_alive?).to be_falsey
+    end
+
+    it "should wait for in use connections to terminate" do
+      expect(subject).to have_received(:wait_for_in_use_connections).once
+      expect(subject).to have_received(:in_use_connections).twice
+    end
+  end
+
+  describe "connection management" do
+    before(:each) { subject.start }
+    context "with only one URL in the list" do
+      it "should use the only URL in 'with_connection'" do
+        subject.with_connection do |c|
+          expect(c).to eq(initial_urls.first)
+        end
+      end
+    end
+
+    context "with multiple URLs in the list" do
+      before :each do
+        allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
+      end
+      let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
+
+      it "should minimize the number of connections to a single URL" do
+        connected_urls = []
+
+        # If we make 2x the number requests as we have URLs we should
+        # connect to each URL exactly 2 times
+        (initial_urls.size*2).times do
+          u, meta = subject.get_connection
+          connected_urls << u
+        end
+
+        connected_urls.each {|u| subject.return_connection(u) }
+        initial_urls.each do |url|
+          conn_count = connected_urls.select {|u| u == url}.size
+          expect(conn_count).to eql(2)
+        end
+      end
+
+      it "should correctly resurrect the dead" do
+        u,m = subject.get_connection
+
+        # The resurrectionist will call this to check on the backend
+        response = double("response")
+        expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
+
+        subject.return_connection(u)
+        subject.mark_dead(u, Exception.new)
+
+        expect(subject.url_meta(u)[:state]).to eql(:dead)
+        sleep subject.resurrect_delay + 1
+        expect(subject.url_meta(u)[:state]).to eql(:alive)
+      end
+    end
+  end
+
+  describe "version tracking" do
+    let(:initial_urls) { [
+      ::LogStash::Util::SafeURI.new("http://somehost:9200"),
+      ::LogStash::Util::SafeURI.new("http://otherhost:9201")
+    ] }
+
+    before(:each) do
+      allow(subject).to receive(:perform_request_to_url).and_return(nil)
+      subject.start
+    end
+
+    it "picks the largest major version" do
+      expect(subject.maximum_seen_major_version).to eq(7)
+    end
+
+    context "if there are nodes with multiple major versions" do
+      let(:node_versions) { [ "0.0.0", "7.0.0" ] }
+      it "picks the largest major version" do
+        expect(subject.maximum_seen_major_version).to eq(7)
+      end
+    end
+  end
+  describe "distribution checking" do
+    before(:each) do
+      allow(subject).to receive(:health_check_request)
+    end
+
+    let(:options) do
+      super().merge(:distribution_checker => distribution_checker)
+    end
+
+    context 'when DistributionChecker#is_supported? returns false' do
+      let(:distribution_checker) { double('DistributionChecker', :is_supported? => false) }
+
+      it 'does not mark the URL as active' do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(0)
+      end
+    end
+
+    context 'when DistributionChecker#is_supported? returns true' do
+      let(:distribution_checker) { double('DistributionChecker', :is_supported? => true) }
+
+      it 'marks the URL as active' do
+        subject.update_initial_urls
+        expect(subject.alive_urls_count).to eq(1)
+      end
+    end
+  end
+  describe 'distribution checking with cluster output' do
+    let(:options) do
+      super().merge(:distribution_checker => LogStash::Outputs::OpenSearch::DistributionChecker.new(logger))
+    end
+
+    before(:each) do
+      allow(subject).to receive(:health_check_request)
+    end
+
+    context 'when using opensearch' do
+
+      context "if cluster doesn't return a valid distribution" do
+        let(:get_distribution) { nil }
+
+        it 'marks the url as dead' do
+          subject.update_initial_urls
+          expect(subject.alive_urls_count).to eq(0)
+        end
+
+        it 'logs message' do
+          expect(subject.distribution_checker).to receive(:log_not_supported).once.and_call_original
+          subject.update_initial_urls
+        end
+      end
+
+      context 'if cluster returns opensearch' do
+        let(:get_distribution) { 'opensearch' }
+
+        it "marks the url as active" do
+          subject.update_initial_urls
+          expect(subject.alive_urls_count).to eq(1)
+        end
+
+        it 'does not log message' do
+          expect(subject.distribution_checker).to_not receive(:log_not_supported)
+          subject.update_initial_urls
+        end
+      end
+
+      context 'if cluster returns oss' do
+        let(:get_distribution) { 'oss' }
+
+        it 'marks the url as active' do
+          subject.update_initial_urls
+          expect(subject.alive_urls_count).to eq(1)
+        end
+
+        it 'does not log message' do
+          expect(subject.distribution_checker).to_not receive(:log_not_supported)
+          subject.update_initial_urls
+        end
+      end
+    end
+  end
+end

data/spec/unit/outputs/opensearch/http_client_spec.rb

@@ -0,0 +1,292 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+
+require_relative "../../../../spec/spec_helper"
+require "logstash/outputs/opensearch/http_client"
+require "cabin"
+
+describe LogStash::Outputs::OpenSearch::HttpClient do
+  let(:ssl) { nil }
+  let(:base_options) do
+    opts = {
+      :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
+      :logger => Cabin::Channel.get,
+      :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy)
+    }
+
+    if !ssl.nil? # Shortcut to set this
+      opts[:client_settings] = {:ssl => {:enabled => ssl}}
+    end
+
+    opts
+  end
+
+  describe "Host/URL Parsing" do
+    subject { described_class.new(base_options) }
+
+    let(:true_hostname) { "my-dash.hostname" }
+    let(:ipv6_hostname) { "[::1]" }
+    let(:ipv4_hostname) { "127.0.0.1" }
+    let(:port) { 9202 }
+    let(:hostname_port) { "#{hostname}:#{port}" }
+    let(:hostname_port_uri) { ::LogStash::Util::SafeURI.new("//#{hostname_port}") }
+    let(:http_hostname_port) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}") }
+    let(:https_hostname_port) { ::LogStash::Util::SafeURI.new("https://#{hostname_port}") }
+    let(:http_hostname_port_path) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}/path") }
+    
+    shared_examples("proper host handling") do
+      it "should properly transform a host:port string to a URL" do
+        expect(subject.host_to_url(hostname_port_uri).to_s).to eq(http_hostname_port.to_s + "/")
+      end
+
+      it "should not raise an error with a / for a path" do
+        expect(subject.host_to_url(::LogStash::Util::SafeURI.new("#{http_hostname_port}/"))).to eq(LogStash::Util::SafeURI.new("#{http_hostname_port}/"))
+      end
+
+      it "should parse full URLs correctly" do
+        expect(subject.host_to_url(http_hostname_port).to_s).to eq(http_hostname_port.to_s + "/")
+      end
+
+      describe "ssl" do
+        context "when SSL is true" do
+          let(:ssl) { true }
+          let(:base_options) { super().merge(:hosts => [http_hostname_port]) }
+
+          it "should refuse to handle an http url" do
+            expect {
+              subject.host_to_url(http_hostname_port)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+
+        context "when SSL is false" do
+          let(:ssl) { false }
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
+          
+          it "should refuse to handle an https url" do
+            expect {
+              subject.host_to_url(https_hostname_port)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+
+        describe "ssl is nil" do
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
+          it "should handle an ssl url correctly when SSL is nil" do
+            subject
+            expect(subject.host_to_url(https_hostname_port).to_s).to eq(https_hostname_port.to_s + "/")
+          end
+        end       
+      end
+
+      describe "path" do
+        let(:url) { http_hostname_port_path }
+        let(:base_options) { super().merge(:hosts => [url]) }
+        
+        it "should allow paths in a url" do
+          expect(subject.host_to_url(url)).to eq(url)
+        end
+
+        context "with the path option set" do
+          let(:base_options) { super().merge(:client_settings => {:path => "/otherpath"}) }
+
+          it "should not allow paths in two places" do
+            expect {
+              subject.host_to_url(url)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+        
+        context "with a path missing a leading /" do
+          let(:url) { http_hostname_port }
+          let(:base_options) { super().merge(:client_settings => {:path => "otherpath"}) }
+          
+          
+          it "should automatically insert a / in front of path overlays" do
+            expected = url.clone
+            expected.path = url.path + "/otherpath"
+            expect(subject.host_to_url(url)).to eq(expected)
+          end
+        end
+      end
+    end
+
+    describe "an regular hostname" do
+      let(:hostname) { true_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv4 host" do
+      let(:hostname) { ipv4_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv6 host" do
+      let(:hostname) { ipv6_hostname }
+      include_examples("proper host handling")
+    end
+  end
+
+  describe "get" do
+    subject { described_class.new(base_options) }
+    let(:body) { "foobar" }
+    let(:path) { "/hello-id" }
+    let(:get_response) {
+      double("response", :body => LogStash::Json::dump( { "body" => body }))
+    }
+
+    it "returns the hash response" do
+      expect(subject.pool).to receive(:get).with(path, nil).and_return(get_response)
+      expect(subject.get(path)["body"]).to eq(body)
+    end
+  end
+
+  describe "index template" do
+    subject { described_class.new(base_options) }
+    let(:template_name) { "logstash" }
+    let(:template) { {} }
+    let(:get_response) {
+      double("response", :body => {})
+    }
+
+    it "should call index template" do
+      expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
+      subject.template_put(template_name, template)
+    end
+  end
+
+  describe "join_bulk_responses" do
+    subject { described_class.new(base_options) }
+
+    context "when items key is available" do
+      require "json"
+      let(:bulk_response) {
+        LogStash::Json.load ('[{
+          "items": [{
+            "delete": {
+              "_index":   "website",
+              "_type":    "blog",
+              "_id":      "123",
+              "_version": 2,
+              "status":   200,
+              "found":    true
+            }
+          }],
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 1
+      end
+    end
+
+    context "when items key is not available" do
+      require "json"
+      let(:bulk_response) {
+        JSON.parse ('[{
+          "took": 4,
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 0
+      end
+    end
+  end
+
+  describe "#bulk" do
+    subject(:http_client) { described_class.new(base_options) }
+
+    require "json"
+    let(:message) { "hey" }
+    let(:actions) { [
+      ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
+    ]}
+
+    [true,false].each do |http_compression_enabled|
+      context "with `http_compression => #{http_compression_enabled}`" do
+
+        let(:base_options) { super().merge(:client_settings => {:http_compression => http_compression_enabled}) }
+
+        before(:each) do
+          if http_compression_enabled
+            expect(http_client).to receive(:gzip_writer).at_least(:once).and_call_original
+          else
+            expect(http_client).to_not receive(:gzip_writer)
+          end
+        end
+
+        context "if a message is over TARGET_BULK_BYTES" do
+          let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+          let(:message) { "a" * (target_bulk_bytes + 1) }
+
+          it "should be handled properly" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once do |data|
+              if !http_compression_enabled
+                expect(data.size).to be > target_bulk_bytes
+              else
+                expect(Zlib::gunzip(data.string).size).to be > target_bulk_bytes
+              end
+            end
+            s = subject.send(:bulk, actions)
+          end
+        end
+
+        context "with two messages" do
+          let(:message1) { "hey" }
+          let(:message2) { "you" }
+          let(:actions) { [
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+          ]}
+          it "executes one bulk_send operation" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once
+            s = subject.send(:bulk, actions)
+          end
+
+          context "if one exceeds TARGET_BULK_BYTES" do
+            let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+            let(:message1) { "a" * (target_bulk_bytes + 1) }
+            it "executes two bulk_send operations" do
+              allow(subject).to receive(:join_bulk_responses)
+              expect(subject).to receive(:bulk_send).twice
+              s = subject.send(:bulk, actions)
+            end
+          end
+        end
+       end
+     end
+  end
+
+  describe "sniffing" do
+    let(:client) { LogStash::Outputs::OpenSearch::HttpClient.new(base_options.merge(client_opts)) }
+
+    context "with sniffing enabled" do
+      let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
+
+      it "should start the sniffer" do
+        expect(client.pool.sniffing).to be_truthy
+      end
+    end
+
+    context "with sniffing disabled" do
+      let(:client_opts) { {:sniffing => false} }
+
+      it "should not start the sniffer" do
+        expect(client.pool.sniffing).to be_falsey
+      end
+    end
+  end
+end