logstash-output-elasticsearch-test 10.3.0-x86_64-linux

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -0,0 +1,274 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "json"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
+  let(:logger) { Cabin::Channel.get }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
+  let(:es_node_versions) { [ "0.0.0" ] }
+  let(:oss) { true }
+  let(:valid_license) { true }
+
+  subject { described_class.new(logger, adapter, initial_urls, options) }
+
+  let(:manticore_double) { double("manticore a") }
+  before(:each) do
+
+    allow(::LogStash::Outputs::ElasticSearch).to receive(:oss?).and_return(oss)
+    response_double = double("manticore response").as_null_object
+    # Allow healtchecks
+    allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:close)
+
+    allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
+
+    allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
+    allow(subject).to receive(:oss?).and_return(oss)
+    allow(subject).to receive(:valid_es_license?).and_return(valid_license)
+  end
+
+  after do
+    subject.close
+  end
+
+  describe "initialization" do
+    it "should be successful" do
+      expect { subject }.not_to raise_error
+      subject.start
+    end
+  end
+
+  describe "the resurrectionist" do
+    before(:each) { subject.start }
+    it "should start the resurrectionist when created" do
+      expect(subject.resurrectionist_alive?).to eql(true)
+    end
+
+    it "should attempt to resurrect connections after the ressurrect delay" do
+      expect(subject).to receive(:healthcheck!).once
+      sleep(subject.resurrect_delay + 1)
+    end
+
+    describe "healthcheck url handling" do
+      let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+
+      context "and not setting healthcheck_path" do
+        it "performs the healthcheck to the root" do
+          expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
+            expect(method).to eq(:head)
+            expect(url.path).to be_empty
+            expect(req_path).to eq("/")
+          end
+          subject.healthcheck!
+        end
+      end
+
+      context "and setting healthcheck_path" do
+        let(:healthcheck_path) { "/my/health" }
+        let(:options) { super.merge(:healthcheck_path => healthcheck_path) }
+        it "performs the healthcheck to the healthcheck_path" do
+          expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
+            expect(method).to eq(:head)
+            expect(url.path).to be_empty
+            expect(req_path).to eq(healthcheck_path)
+          end
+          subject.healthcheck!
+        end
+      end
+    end
+  end
+
+  describe 'resolving the address from Elasticsearch node info' do
+    let(:host) { "node.elastic.co"}
+    let(:ip_address) { "192.168.1.0"}
+    let(:port) { 9200 }
+
+    context 'in Elasticsearch 1.x format' do
+      context 'with host and ip address' do
+        let(:publish_address) { "inet[#{host}/#{ip_address}:#{port}]"}
+        it 'should correctly extract the host' do
+          expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
+        end
+      end
+      context 'with ip address' do
+        let(:publish_address) { "inet[/#{ip_address}:#{port}]"}
+        it 'should correctly extract the ip address' do
+          expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
+        end
+      end
+    end
+
+    context 'in Elasticsearch 2.x-6.x format' do
+      let(:publish_address) { "#{ip_address}:#{port}"}
+      it 'should correctly extract the ip address' do
+        expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("//#{ip_address}:#{port}"))
+      end
+    end
+
+    context 'in Elasticsearch 7.x'
+    context 'with host and ip address' do
+      let(:publish_address) { "#{host}/#{ip_address}:#{port}"}
+      it 'should correctly extract the host' do
+        expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
+      end
+    end
+    context 'with ip address' do
+      let(:publish_address) { "#{ip_address}:#{port}"}
+      it 'should correctly extract the ip address' do
+        expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
+      end
+    end
+  end
+
+  describe "the sniffer" do
+    before(:each) { subject.start }
+    it "should not start the sniffer by default" do
+      expect(subject.sniffer_alive?).to eql(nil)
+    end
+
+    context "when enabled" do
+      let(:options) { super.merge(:sniffing => true)}
+
+      it "should start the sniffer" do
+        expect(subject.sniffer_alive?).to eql(true)
+      end
+    end
+  end
+
+  describe "closing" do
+    before do
+      subject.start
+      # Simulate a single in use connection on the first check of this
+      allow(adapter).to receive(:close).and_call_original
+      allow(subject).to receive(:wait_for_in_use_connections).and_call_original
+      allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
+      allow(subject).to receive(:start)
+      subject.close
+    end
+
+    it "should close the adapter" do
+      expect(adapter).to have_received(:close)
+    end
+
+    it "should stop the resurrectionist" do
+      expect(subject.resurrectionist_alive?).to eql(false)
+    end
+
+    it "should stop the sniffer" do
+      # If no sniffer (the default) returns nil
+      expect(subject.sniffer_alive?).to be_falsey
+    end
+
+    it "should wait for in use connections to terminate" do
+      expect(subject).to have_received(:wait_for_in_use_connections).once
+      expect(subject).to have_received(:in_use_connections).twice
+    end
+  end
+
+  describe "connection management" do
+    before(:each) { subject.start }
+    context "with only one URL in the list" do
+      it "should use the only URL in 'with_connection'" do
+        subject.with_connection do |c|
+          expect(c).to eq(initial_urls.first)
+        end
+      end
+    end
+
+    context "with multiple URLs in the list" do
+      before :each do
+        allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
+      end
+      let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
+
+      it "should minimize the number of connections to a single URL" do
+        connected_urls = []
+
+        # If we make 2x the number requests as we have URLs we should
+        # connect to each URL exactly 2 times
+        (initial_urls.size*2).times do
+          u, meta = subject.get_connection
+          connected_urls << u
+        end
+
+        connected_urls.each {|u| subject.return_connection(u) }
+        initial_urls.each do |url|
+          conn_count = connected_urls.select {|u| u == url}.size
+          expect(conn_count).to eql(2)
+        end
+      end
+
+      it "should correctly resurrect the dead" do
+        u,m = subject.get_connection
+
+        # The resurrectionist will call this to check on the backend
+        response = double("response")
+        expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
+
+        subject.return_connection(u)
+        subject.mark_dead(u, Exception.new)
+
+        expect(subject.url_meta(u)[:state]).to eql(:dead)
+        sleep subject.resurrect_delay + 1
+        expect(subject.url_meta(u)[:state]).to eql(:alive)
+      end
+    end
+  end
+
+  describe "version tracking" do
+    let(:initial_urls) { [
+      ::LogStash::Util::SafeURI.new("http://somehost:9200"),
+      ::LogStash::Util::SafeURI.new("http://otherhost:9201")
+    ] }
+
+    before(:each) do
+      allow(subject).to receive(:perform_request_to_url).and_return(nil)
+      subject.start
+    end
+
+    it "picks the largest major version" do
+      expect(subject.maximum_seen_major_version).to eq(0)
+    end
+
+    context "if there are nodes with multiple major versions" do
+      let(:es_node_versions) { [ "0.0.0", "6.0.0" ] }
+      it "picks the largest major version" do
+        expect(subject.maximum_seen_major_version).to eq(6)
+      end
+    end
+  end
+
+  describe "license checking" do
+    before(:each) do
+      allow(subject).to receive(:health_check_request)
+    end
+    context "when using default logstash distribution" do
+      let(:oss) { false }
+      context "if ES doesn't return a valid license" do
+        let(:valid_license) { false }
+        it "marks the url as active" do
+          subject.update_initial_urls
+          expect(subject.alive_urls_count).to eq(1)
+        end
+        it "logs a warning" do
+          expect(subject).to receive(:log_license_deprecation_warn).once
+          subject.update_initial_urls
+        end
+      end
+      context "if ES returns a valid license" do
+        let(:valid_license) { true }
+        it "marks the url as active" do
+          subject.update_initial_urls
+          expect(subject.alive_urls_count).to eq(1)
+        end
+        it "does not log a warning" do
+          expect(subject).to_not receive(:log_license_deprecation_warn)
+          subject.update_initial_urls
+        end
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -0,0 +1,250 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "java"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient do
+  let(:ssl) { nil }
+  let(:base_options) do
+    opts = {
+      :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
+      :logger => Cabin::Channel.get,
+      :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy)
+    }
+
+    if !ssl.nil? # Shortcut to set this
+      opts[:client_settings] = {:ssl => {:enabled => ssl}}
+    end
+
+    opts
+  end
+
+  describe "Host/URL Parsing" do
+    subject { described_class.new(base_options) }
+
+    let(:true_hostname) { "my-dash.hostname" }
+    let(:ipv6_hostname) { "[::1]" }
+    let(:ipv4_hostname) { "127.0.0.1" }
+    let(:port) { 9202 }
+    let(:hostname_port) { "#{hostname}:#{port}" }
+    let(:hostname_port_uri) { ::LogStash::Util::SafeURI.new("//#{hostname_port}") }
+    let(:http_hostname_port) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}") }
+    let(:https_hostname_port) { ::LogStash::Util::SafeURI.new("https://#{hostname_port}") }
+    let(:http_hostname_port_path) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}/path") }
+    
+    shared_examples("proper host handling") do
+      it "should properly transform a host:port string to a URL" do
+        expect(subject.host_to_url(hostname_port_uri).to_s).to eq(http_hostname_port.to_s + "/")
+      end
+
+      it "should not raise an error with a / for a path" do
+        expect(subject.host_to_url(::LogStash::Util::SafeURI.new("#{http_hostname_port}/"))).to eq(LogStash::Util::SafeURI.new("#{http_hostname_port}/"))
+      end
+
+      it "should parse full URLs correctly" do
+        expect(subject.host_to_url(http_hostname_port).to_s).to eq(http_hostname_port.to_s + "/")
+      end
+
+      describe "ssl" do
+        context "when SSL is true" do
+          let(:ssl) { true }
+          let(:base_options) { super.merge(:hosts => [http_hostname_port]) }
+
+          it "should refuse to handle an http url" do
+            expect {
+              subject.host_to_url(http_hostname_port)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+
+        context "when SSL is false" do
+          let(:ssl) { false }
+          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          
+          it "should refuse to handle an https url" do
+            expect {
+              subject.host_to_url(https_hostname_port)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+
+        describe "ssl is nil" do
+          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          it "should handle an ssl url correctly when SSL is nil" do
+            subject
+            expect(subject.host_to_url(https_hostname_port).to_s).to eq(https_hostname_port.to_s + "/")
+          end
+        end       
+      end
+
+      describe "path" do
+        let(:url) { http_hostname_port_path }
+        let(:base_options) { super.merge(:hosts => [url]) }
+        
+        it "should allow paths in a url" do
+          expect(subject.host_to_url(url)).to eq(url)
+        end
+
+        context "with the path option set" do
+          let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) }
+
+          it "should not allow paths in two places" do
+            expect {
+              subject.host_to_url(url)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+        
+        context "with a path missing a leading /" do
+          let(:url) { http_hostname_port }
+          let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) }
+          
+          
+          it "should automatically insert a / in front of path overlays" do
+            expected = url.clone
+            expected.path = url.path + "/otherpath"
+            expect(subject.host_to_url(url)).to eq(expected)
+          end
+        end
+      end
+    end
+
+    describe "an regular hostname" do
+      let(:hostname) { true_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv4 host" do
+      let(:hostname) { ipv4_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv6 host" do
+      let(:hostname) { ipv6_hostname }
+      include_examples("proper host handling")
+    end
+  end
+
+  describe "get" do
+    subject { described_class.new(base_options) }
+    let(:body) { "foobar" }
+    let(:path) { "/hello-id" }
+    let(:get_response) {
+      double("response", :body => LogStash::Json::dump( { "body" => body }))
+    }
+
+    it "returns the hash response" do
+      expect(subject.pool).to receive(:get).with(path, nil).and_return(get_response)
+      expect(subject.get(path)["body"]).to eq(body)
+    end
+  end
+
+  describe "join_bulk_responses" do
+    subject { described_class.new(base_options) }
+
+    context "when items key is available" do
+      require "json"
+      let(:bulk_response) {
+        LogStash::Json.load ('[{
+          "items": [{
+            "delete": {
+              "_index":   "website",
+              "_type":    "blog",
+              "_id":      "123",
+              "_version": 2,
+              "status":   200,
+              "found":    true
+            }
+          }],
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 1
+      end
+    end
+
+    context "when items key is not available" do
+      require "json"
+      let(:bulk_response) {
+        JSON.parse ('[{
+          "took": 4,
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 0
+      end
+    end
+  end
+
+  describe "#bulk" do
+    subject { described_class.new(base_options) }
+
+    require "json"
+    let(:message) { "hey" }
+    let(:actions) { [
+      ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
+    ]}
+
+    context "if a message is over TARGET_BULK_BYTES" do
+      let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+      let(:message) { "a" * (target_bulk_bytes + 1) }
+
+      it "should be handled properly" do
+        allow(subject).to receive(:join_bulk_responses)
+        expect(subject).to receive(:bulk_send).once do |data|
+          expect(data.size).to be > target_bulk_bytes
+        end
+        s = subject.send(:bulk, actions)
+      end
+    end
+
+    context "with two messages" do
+      let(:message1) { "hey" }
+      let(:message2) { "you" }
+      let(:actions) { [
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+      ]}
+      it "executes one bulk_send operation" do
+        allow(subject).to receive(:join_bulk_responses)
+        expect(subject).to receive(:bulk_send).once
+        s = subject.send(:bulk, actions)
+      end
+
+      context "if one exceeds TARGET_BULK_BYTES" do
+        let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+        let(:message1) { "a" * (target_bulk_bytes + 1) }
+        it "executes two bulk_send operations" do
+          allow(subject).to receive(:join_bulk_responses)
+          expect(subject).to receive(:bulk_send).twice
+          s = subject.send(:bulk, actions)
+        end
+      end
+    end
+  end
+
+  describe "sniffing" do
+    let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
+
+    context "with sniffing enabled" do
+      let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
+
+      it "should start the sniffer" do
+        expect(client.pool.sniffing).to be_truthy
+      end
+    end
+
+    context "with sniffing disabled" do
+      let(:client_opts) { {:sniffing => false} }
+
+      it "should not start the sniffer" do
+        expect(client.pool.sniffing).to be_falsey
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -0,0 +1,25 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "java"
+require "json"
+
+describe LogStash::Outputs::ElasticSearch::TemplateManager do
+
+  describe ".default_template_path" do
+    context "elasticsearch 1.x" do
+      it "chooses the 2x template" do
+        expect(described_class.default_template_path(1)).to match(/elasticsearch-template-es2x.json/)
+      end
+    end
+    context "elasticsearch 2.x" do
+      it "chooses the 2x template" do
+        expect(described_class.default_template_path(2)).to match(/elasticsearch-template-es2x.json/)
+      end
+    end
+    context "elasticsearch 5.x" do
+      it "chooses the 5x template" do
+        expect(described_class.default_template_path(5)).to match(/elasticsearch-template-es5x.json/)
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -0,0 +1,72 @@
+require_relative "../../../spec/es_spec_helper"
+require 'stud/temporary'
+require "logstash/outputs/elasticsearch"
+require 'manticore/client'
+
+describe "Proxy option" do
+  let(:settings) { { "hosts" => "node01" } }
+  subject {
+    LogStash::Outputs::ElasticSearch.new(settings)
+  }
+
+  before do
+    allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
+  end
+
+  describe "valid configs" do
+    before do
+      subject.register
+    end
+
+    after do
+      subject.close
+    end
+
+    context "when specified as a URI" do
+      shared_examples("hash conversion") do |hash|
+        let(:settings) { super.merge("proxy" => proxy)}
+        
+        it "should set the proxy to the correct hash value" do
+          expect(::Manticore::Client).to have_received(:new) do |options|
+            expect(options[:proxy]).to eq(hash)
+          end
+        end
+      end
+      
+      describe "simple proxy" do
+        let(:proxy) { LogStash::Util::SafeURI.new("http://127.0.0.1:1234") }
+
+        include_examples("hash conversion",
+          {
+            :host => "127.0.0.1",
+            :scheme => "http",
+            :port => 1234  
+          }
+        )
+      end
+      
+      
+      describe "a secure authed proxy" do
+        let(:proxy) { LogStash::Util::SafeURI.new("https://myuser:mypass@127.0.0.1:1234") }
+
+        include_examples("hash conversion",
+          {
+            :host => "127.0.0.1",
+            :scheme => "https",
+            :user => "myuser",
+            :password => "mypass",
+            :port => 1234  
+          }
+        )
+      end
+    end
+
+    context "when not specified" do
+      it "should not send the proxy option to manticore" do
+        expect(::Manticore::Client).to have_received(:new) do |options|
+          expect(options).not_to include(:proxy)
+        end
+      end
+    end
+  end
+end