logstash-output-amazon_es 2.0.1-java → 6.4.0-java

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -0,0 +1,105 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/amazon_es/http_client"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
+  let(:logger) { Cabin::Channel.get }
+  let(:options) { {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA',
+                   :aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'} }
+
+  subject { described_class.new(logger, options) }
+
+  it "should raise an exception if requests are issued after close" do
+    subject.close
+    expect { subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
+  end
+
+  it "should implement host unreachable exceptions" do
+    expect(subject.host_unreachable_exceptions).to be_a(Array)
+  end
+
+
+  describe "bad response codes" do
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+
+    it "should raise a bad response code error" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(500)
+      allow(resp).to receive(:body).and_return("a body")
+
+      expect(subject.manticore).to receive(:get).
+        with(uri.to_s + "/", anything).
+        and_return(resp)
+
+      uri_with_path = uri.clone
+      uri_with_path.path = "/"
+
+      expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
+        with(resp.code, uri_with_path, nil, resp.body).and_call_original
+
+      expect do
+        subject.perform_request(uri, :get, "/")
+      end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
+    end
+  end
+
+  describe "format_url" do
+    let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
+    let(:path) { "_bulk" }
+    subject { described_class.new(double("logger"),
+                                  {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA',
+                                   :aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'} ) }
+
+    it "should add the path argument to the uri's path" do
+      expect(subject.format_url(url, path).path).to eq("/path/_bulk")
+    end
+
+    context "when uri contains query parameters" do
+      let(:query_params) { "query=value&key=value2" }
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
+      let(:formatted) { subject.format_url(url, path)}
+
+
+      it "should retain query_params after format" do
+        expect(formatted.query).to eq(query_params)
+      end
+      
+      context "and the path contains query parameters" do
+        let(:path) { "/special_path?specialParam=123" }
+
+        it "should join the query correctly" do
+          expect(formatted.query).to eq(query_params + "&specialParam=123")
+        end
+      end
+    end
+    
+    context "when the path contains query parameters" do
+      let(:path) { "/special_bulk?pathParam=1"}
+      let(:formatted) { subject.format_url(url, path) }
+      
+      it "should add the path correctly" do
+        expect(formatted.path).to eq("#{url.path}special_bulk")
+      end 
+      
+      it "should add the query parameters correctly" do
+        expect(formatted.query).to eq("pathParam=1")
+      end
+    end
+
+    context "when uri contains credentials" do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it "should remove credentials after format" do
+        expect(formatted.userinfo).to be_nil
+      end
+    end
+  end
+
+  describe "integration specs", :integration => true do
+    it "should perform correct tests without error" do
+      resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")
+      expect(resp.code).to eql(200)
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -0,0 +1,198 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/amazon_es/http_client"
+require "json"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
+  let(:logger) { Cabin::Channel.get }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger,
+                                                                                     {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA',
+                                                                                              :aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'}) }
+  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
+  let(:es_node_versions) { [ "0.0.0" ] }
+
+  subject { described_class.new(logger, adapter, initial_urls, options) }
+
+  let(:manticore_double) { double("manticore a") }
+  before(:each) do
+
+    response_double = double("manticore response").as_null_object
+    # Allow healtchecks
+    allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:close)
+
+    allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
+
+    allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions)
+  end
+
+  after do
+    subject.close
+  end
+
+  describe "initialization" do
+    it "should be successful" do
+      expect { subject }.not_to raise_error
+      subject.start
+    end
+  end
+
+  describe "the resurrectionist" do
+    before(:each) { subject.start }
+    it "should start the resurrectionist when created" do
+      expect(subject.resurrectionist_alive?).to eql(true)
+    end
+
+    it "should attempt to resurrect connections after the ressurrect delay" do
+      expect(subject).to receive(:healthcheck!).once
+      sleep(subject.resurrect_delay + 1)
+    end
+
+    describe "healthcheck url handling" do
+      let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+
+      context "and not setting healthcheck_path" do
+        it "performs the healthcheck to the root" do
+          expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
+            expect(method).to eq(:head)
+            expect(url.path).to be_empty
+            expect(req_path).to eq("/")
+          end
+          subject.healthcheck!
+        end
+      end
+
+      context "and setting healthcheck_path" do
+        let(:healthcheck_path) { "/my/health" }
+        let(:options) { super.merge(:healthcheck_path => healthcheck_path) }
+        it "performs the healthcheck to the healthcheck_path" do
+          expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
+            expect(method).to eq(:head)
+            expect(url.path).to be_empty
+            expect(req_path).to eq(healthcheck_path)
+          end
+          subject.healthcheck!
+        end
+      end
+    end
+  end
+
+  describe "the sniffer" do
+    before(:each) { subject.start }
+    it "should not start the sniffer by default" do
+      expect(subject.sniffer_alive?).to eql(nil)
+    end
+
+    context "when enabled" do
+      let(:options) { super.merge(:sniffing => true)}
+
+      it "should start the sniffer" do
+        expect(subject.sniffer_alive?).to eql(true)
+      end
+    end
+  end
+
+  describe "closing" do
+    before do
+      subject.start
+      # Simulate a single in use connection on the first check of this
+      allow(adapter).to receive(:close).and_call_original
+      allow(subject).to receive(:wait_for_in_use_connections).and_call_original
+      allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
+      allow(subject).to receive(:start)
+      subject.close
+    end
+
+    it "should close the adapter" do
+      expect(adapter).to have_received(:close)
+    end
+
+    it "should stop the resurrectionist" do
+      expect(subject.resurrectionist_alive?).to eql(false)
+    end
+
+    it "should stop the sniffer" do
+      # If no sniffer (the default) returns nil
+      expect(subject.sniffer_alive?).to be_falsey
+    end
+
+    it "should wait for in use connections to terminate" do
+      expect(subject).to have_received(:wait_for_in_use_connections).once
+      expect(subject).to have_received(:in_use_connections).twice
+    end
+  end
+
+  describe "connection management" do
+    before(:each) { subject.start }
+    context "with only one URL in the list" do
+      it "should use the only URL in 'with_connection'" do
+        subject.with_connection do |c|
+          expect(c).to eq(initial_urls.first)
+        end
+      end
+    end
+
+    context "with multiple URLs in the list" do
+      before :each do
+        allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
+      end
+      let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200") ] }
+
+      it "should minimize the number of connections to a single URL" do
+        connected_urls = []
+
+        # If we make 2x the number requests as we have URLs we should
+        # connect to each URL exactly 2 times
+        (initial_urls.size*2).times do
+          u, meta = subject.get_connection
+          connected_urls << u
+        end
+
+        connected_urls.each {|u| subject.return_connection(u) }
+        initial_urls.each do |url|
+          conn_count = connected_urls.select {|u| u == url}.size
+          expect(conn_count).to eql(2)
+        end
+      end
+
+      it "should correctly resurrect the dead" do
+        u,m = subject.get_connection
+
+        # The resurrectionist will call this to check on the backend
+        response = double("response")
+        expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
+
+        subject.return_connection(u)
+        subject.mark_dead(u, Exception.new)
+
+        expect(subject.url_meta(u)[:state]).to eql(:dead)
+        sleep subject.resurrect_delay + 1
+        expect(subject.url_meta(u)[:state]).to eql(:alive)
+      end
+    end
+  end
+
+  describe "version tracking" do
+    let(:initial_urls) { [
+      ::LogStash::Util::SafeURI.new("http://somehost:9200"),
+      ::LogStash::Util::SafeURI.new("http://otherhost:9201")
+    ] }
+
+    before(:each) do
+      allow(subject).to receive(:perform_request_to_url).and_return(nil)
+      subject.start
+    end
+
+    it "picks the largest major version" do
+      expect(subject.maximum_seen_major_version).to eq(0)
+    end
+
+    context "if there are nodes with multiple major versions" do
+      let(:es_node_versions) { [ "0.0.0", "6.0.0" ] }
+      it "picks the largest major version" do
+        expect(subject.maximum_seen_major_version).to eq(6)
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -0,0 +1,222 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/amazon_es/http_client"
+require "java"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient do
+  let(:ssl) { nil }
+  let(:base_options) do
+    opts = {
+      :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
+      :logger => Cabin::Channel.get,
+      :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy),
+      :protocol => "http",
+      :port => 9200,
+      :aws_access_key_id => "AAAAAAAAAAAAAAAAAAAA",
+      :aws_secret_access_key => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+    }
+
+    if !ssl.nil? # Shortcut to set this
+      opts[:client_settings] = {:ssl => {:enabled => ssl}}
+    end
+
+    opts
+  end
+
+  describe "Host/URL Parsing" do
+    subject { described_class.new(base_options) }
+
+    let(:true_hostname) { "my-dash.hostname" }
+    let(:ipv6_hostname) { "[::1]" }
+    let(:ipv4_hostname) { "127.0.0.1" }
+    let(:port) { 9200 }
+    let(:protocol) {"http"}
+    let(:aws_access_key_id) {"AAAAAAAAAAAAAAAAAAAA"}
+    let(:aws_secret_access_key) {"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
+    let(:hostname_port) { "#{hostname}:#{port}" }
+    let(:hostname_port_uri) { ::LogStash::Util::SafeURI.new("//#{hostname_port}") }
+    let(:http_hostname_port) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}") }
+    let(:https_hostname_port) { ::LogStash::Util::SafeURI.new("https://#{hostname_port}") }
+    let(:http_hostname_port_path) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}/path") }
+    
+    shared_examples("proper host handling") do
+      it "should properly transform a host:port string to a URL" do
+        expect(subject.host_to_url(hostname_port_uri).to_s).to eq(http_hostname_port.to_s + "/")
+      end
+
+      it "should not raise an error with a / for a path" do
+        expect(subject.host_to_url(::LogStash::Util::SafeURI.new("#{http_hostname_port}/"))).to eq(LogStash::Util::SafeURI.new("#{http_hostname_port}/"))
+      end
+
+      it "should parse full URLs correctly" do
+        expect(subject.host_to_url(http_hostname_port).to_s).to eq(http_hostname_port.to_s + "/")
+      end
+
+
+      describe "path" do
+        let(:url) { http_hostname_port_path }
+        let(:base_options) { super.merge(:hosts => [url]) }
+        
+        it "should allow paths in a url" do
+          expect(subject.host_to_url(url)).to eq(url)
+        end
+
+        context "with the path option set" do
+          let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) }
+
+          it "should not allow paths in two places" do
+            expect {
+              subject.host_to_url(url)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+        
+        context "with a path missing a leading /" do
+          let(:url) { http_hostname_port }
+          let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) }
+          
+          
+          it "should automatically insert a / in front of path overlays" do
+            expected = url.clone
+            expected.path = url.path + "/otherpath"
+            expect(subject.host_to_url(url)).to eq(expected)
+          end
+        end
+      end
+    end
+
+    describe "an regular hostname" do
+      let(:hostname) { true_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv4 host" do
+      let(:hostname) { ipv4_hostname }
+      include_examples("proper host handling")
+    end
+
+  end
+
+  describe "get" do
+    subject { described_class.new(base_options) }
+    let(:body) { "foobar" }
+    let(:path) { "/hello-id" }
+    let(:get_response) {
+      double("response", :body => LogStash::Json::dump( { "body" => body }))
+    }
+
+    it "returns the hash response" do
+      expect(subject.pool).to receive(:get).with(path, nil).and_return(get_response)
+      expect(subject.get(path)["body"]).to eq(body)
+    end
+  end
+
+  describe "join_bulk_responses" do
+    subject { described_class.new(base_options) }
+
+    context "when items key is available" do
+      require "json"
+      let(:bulk_response) {
+        LogStash::Json.load ('[{
+          "items": [{
+            "delete": {
+              "_index":   "website",
+              "_type":    "blog",
+              "_id":      "123",
+              "_version": 2,
+              "status":   200,
+              "found":    true
+            }
+          }],
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 1
+      end
+    end
+
+    context "when items key is not available" do
+      require "json"
+      let(:bulk_response) {
+        JSON.parse ('[{
+          "took": 4,
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 0
+      end
+    end
+  end
+
+  describe "#bulk" do
+    subject { described_class.new(base_options) }
+
+    require "json"
+    let(:message) { "hey" }
+    let(:actions) { [
+      ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
+    ]}
+
+    context "if a message is over TARGET_BULK_BYTES" do
+      let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+      let(:message) { "a" * (target_bulk_bytes + 1) }
+
+      it "should be handled properly" do
+        allow(subject).to receive(:join_bulk_responses)
+        expect(subject).to receive(:bulk_send).once do |data|
+          expect(data.size).to be > target_bulk_bytes
+        end
+        s = subject.send(:bulk, actions)
+      end
+    end
+
+    context "with two messages" do
+      let(:message1) { "hey" }
+      let(:message2) { "you" }
+      let(:actions) { [
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+      ]}
+      it "executes one bulk_send operation" do
+        allow(subject).to receive(:join_bulk_responses)
+        expect(subject).to receive(:bulk_send).once
+        s = subject.send(:bulk, actions)
+      end
+
+      context "if one exceeds TARGET_BULK_BYTES" do
+        let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+        let(:message1) { "a" * (target_bulk_bytes + 1) }
+        it "executes two bulk_send operations" do
+          allow(subject).to receive(:join_bulk_responses)
+          expect(subject).to receive(:bulk_send).twice
+          s = subject.send(:bulk, actions)
+        end
+      end
+    end
+  end
+
+  describe "sniffing" do
+    let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
+
+    context "with sniffing enabled" do
+      let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
+
+      it "should start the sniffer" do
+        expect(client.pool.sniffing).to be_truthy
+      end
+    end
+
+    context "with sniffing disabled" do
+      let(:client_opts) { {:sniffing => false} }
+
+      it "should not start the sniffer" do
+        expect(client.pool.sniffing).to be_falsey
+      end
+    end
+  end
+end