logstash-output-elasticsearch 3.0.2-java → 4.1.0-java

data/spec/integration/outputs/routing_spec.rb

@@ -12,7 +12,7 @@ shared_examples "a routing indexer" do
     before do
       subject.register
       event_count.times do
-        subject.receive(LogStash::Event.new("message" => "Hello World!", "type" => type))
+        subject.multi_receive([LogStash::Event.new("message" => "Hello World!", "type" => type)])
       end
     end
 

data/spec/integration/outputs/secure_spec.rb

@@ -24,8 +24,7 @@ describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure =>
 
   it "sends events to ES" do
     expect {
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
     }.to_not raise_error
   end
 end
@@ -48,8 +47,7 @@ describe "connect using HTTP Authentication", :elasticsearch_secure => true do
 
   it "sends events to ES" do
     expect {
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
     }.to_not raise_error
   end
 end
@@ -78,8 +76,7 @@ describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure =>
 
   it "sends events to ES" do
     expect {
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
     }.to_not raise_error
   end
 end
@@ -101,8 +98,7 @@ describe "connect using HTTP Authentication", :elasticsearch_secure => true do
 
   it "sends events to ES" do
     expect {
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
     }.to_not raise_error
   end
 end

data/spec/integration/outputs/templates_spec.rb

@@ -24,14 +24,16 @@ describe "index template expected behavior", :integration => true do
 
     subject.register
 
-    subject.receive(LogStash::Event.new("message" => "sample message here"))
-    subject.receive(LogStash::Event.new("somevalue" => 100))
-    subject.receive(LogStash::Event.new("somevalue" => 10))
-    subject.receive(LogStash::Event.new("somevalue" => 1))
-    subject.receive(LogStash::Event.new("country" => "us"))
-    subject.receive(LogStash::Event.new("country" => "at"))
-    subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
-    subject.flush
+    subject.multi_receive([
+      LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("somevalue" => 100),
+      LogStash::Event.new("somevalue" => 10),
+      LogStash::Event.new("somevalue" => 1),
+      LogStash::Event.new("country" => "us"),
+      LogStash::Event.new("country" => "at"),
+      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+    ])
+
     @es.indices.refresh
 
     # Wait or fail until everything's indexed.
@@ -88,3 +90,5 @@ describe "index template expected behavior", :integration => true do
     insist { terms }.include?("at")
   end
 end
+
+

data/spec/integration/outputs/update_spec.rb

@@ -2,7 +2,6 @@ require_relative "../../../spec/es_spec_helper"
 
 describe "Update actions", :integration => true do
   require "logstash/outputs/elasticsearch"
-  require "elasticsearch"
 
   def get_es_output( options={} )
     settings = {
@@ -40,16 +39,14 @@ describe "Update actions", :integration => true do
     it "should not create new document" do
       subject = get_es_output({ 'document_id' => "456" } )
       subject.register
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
     end
 
     it "should update existing document" do
       subject = get_es_output({ 'document_id' => "123" })
       subject.register
-      subject.receive(LogStash::Event.new("message" => "updated message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
       insist { r["_source"]["message"] } == 'updated message here'
     end
@@ -59,8 +56,7 @@ describe "Update actions", :integration => true do
     it "should update an existing document that has a 'data' field" do
       subject = get_es_output({ 'document_id' => "123" })
       subject.register
-      subject.receive(LogStash::Event.new("data" => "updated message here", "message" => "foo"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
       insist { r["_source"]["data"] } == 'updated message here'
       insist { r["_source"]["message"] } == 'foo'
@@ -71,8 +67,7 @@ describe "Update actions", :integration => true do
     it "should increment a counter with event/doc 'count' variable" do
       subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
       subject.register
-      subject.receive(LogStash::Event.new("count" => 2))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("count" => 2)])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
       insist { r["_source"]["counter"] } == 3
     end
@@ -80,8 +75,7 @@ describe "Update actions", :integration => true do
     it "should increment a counter with event/doc '[data][count]' nested variable" do
       subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
       subject.register
-      subject.receive(LogStash::Event.new("data" => { "count" => 3 }))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
       insist { r["_source"]["counter"] } == 4
     end
@@ -94,8 +88,7 @@ describe "Update actions", :integration => true do
         'script_type' => 'inline'
       })
       subject.register
-      subject.receive(LogStash::Event.new("counter" => 3 ))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
       insist { r["_source"]["counter"] } == 4
     end
@@ -109,8 +102,7 @@ describe "Update actions", :integration => true do
         'script_type' => 'inline'
       })
       subject.register
-      subject.receive(LogStash::Event.new("counter" => 3 ))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("count" => 3 )])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
       insist { r["_source"]["counter"] } == 4
     end
@@ -124,8 +116,7 @@ describe "Update actions", :integration => true do
         'script_type' => 'inline'
       })
       subject.register
-      subject.receive(LogStash::Event.new("counter" => 3 ))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
       insist { r["_source"]["counter"] } == 3
     end
@@ -139,8 +130,7 @@ describe "Update actions", :integration => true do
         'script_type' => 'indexed'
       })
       subject.register
-      subject.receive(LogStash::Event.new("count" => 4 ))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("count" => 4 )])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
       insist { r["_source"]["counter"] } == 5
     end
@@ -150,8 +140,7 @@ describe "Update actions", :integration => true do
     it "should create new documents with provided upsert" do
       subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
       insist { r["_source"]["message"] } == 'upsert message'
     end
@@ -159,8 +148,7 @@ describe "Update actions", :integration => true do
     it "should create new documents with event/doc as upsert" do
       subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
-      subject.receive(LogStash::Event.new("message" => "sample message here"))
-      subject.flush
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
       insist { r["_source"]["message"] } == 'sample message here'
     end
@@ -169,8 +157,7 @@ describe "Update actions", :integration => true do
       it "should create new documents with upsert content" do
         subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
         subject.register
-        subject.receive(LogStash::Event.new("message" => "sample message here"))
-        subject.flush
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
         r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
         insist { r["_source"]["message"] } == 'upsert message'
       end
@@ -178,8 +165,7 @@ describe "Update actions", :integration => true do
       it "should create new documents with event/doc as script params" do
         subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
         subject.register
-        subject.receive(LogStash::Event.new("counter" => 1))
-        subject.flush
+        subject.multi_receive([LogStash::Event.new("counter" => 1)])
         r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
         insist { r["_source"]["counter"] } == 1
       end

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -0,0 +1,25 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
+  let(:logger) { Cabin::Channel.get }
+  let(:options) { {} }
+
+  subject { described_class.new(logger, options) }
+
+  it "should raise an exception if requests are issued after close" do
+    subject.close
+    expect { subject.perform_request("http://localhost:9200", :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
+  end
+
+  it "should implement host unreachable exceptions" do
+    expect(subject.host_unreachable_exceptions).to be_a(Array)
+  end
+
+  describe "integration specs", :integration => true do
+    it "should perform correct tests without error" do
+      resp = subject.perform_request("http://localhost:9200", :get, "/")
+      expect(resp.code).to eql(200)
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -0,0 +1,142 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "json"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
+  let(:logger) { Cabin::Channel.get }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:initial_urls) { [URI.parse("http://localhost:9200")] }
+  let(:options) { {:resurrect_delay => 2} } # Shorten the delay a bit to speed up tests
+
+  subject { described_class.new(logger, adapter, initial_urls, options) }
+
+  describe "initialization" do
+    it "should be successful" do
+      expect { subject }.not_to raise_error
+    end
+  end
+
+  describe "the resurrectionist" do
+    it "should start the resurrectionist when created" do
+      expect(subject.resurrectionist_alive?).to eql(true)
+    end
+
+    it "should attempt to resurrect connections after the ressurrect delay" do
+      expect(subject).to receive(:resurrect_dead!).once
+      sleep(subject.resurrect_delay + 1)
+    end
+  end
+
+  describe "the sniffer" do
+    it "should not start the sniffer by default" do
+      expect(subject.sniffer_alive?).to eql(nil)
+    end
+
+    context "when enabled" do
+      let(:options) { super.merge(:sniffing => true)}
+
+      it "should start the sniffer" do
+        expect(subject.sniffer_alive?).to eql(true)
+      end
+    end
+
+    describe "check sniff" do
+      context "with a good sniff result" do
+        let(:sniff_resp_path) { File.dirname(__FILE__) + '/../../../../fixtures/5x_node_resp.json' }
+        let(:sniff_resp) { double("resp") }
+        let(:sniff_resp_body) { File.open(sniff_resp_path).read }
+        
+        before do
+          allow(subject).to receive(:perform_request).
+                              with(:get, '_nodes').
+                              and_return([double('url'), sniff_resp])
+          allow(sniff_resp).to receive(:body).and_return(sniff_resp_body)
+        end
+        
+        it "should execute a sniff without error" do
+          expect { subject.check_sniff }.not_to raise_error
+        end
+
+        it "should return the correct sniff URL list" do
+          url_strs = subject.check_sniff.map(&:to_s)
+          expect(url_strs).to include("http://127.0.0.1:9200")
+          expect(url_strs).to include("http://127.0.0.1:9201")
+        end
+      end
+    end
+  end
+
+  describe "closing" do
+    before do
+      # Simulate a single in use connection on the first check of this
+      allow(adapter).to receive(:close).and_call_original
+      allow(subject).to receive(:wait_for_in_use_connections).and_call_original
+      allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
+      subject.close
+    end
+
+    it "should close the adapter" do
+      expect(adapter).to have_received(:close)
+    end
+
+    it "should stop the resurrectionist" do
+      expect(subject.resurrectionist_alive?).to eql(false)
+    end
+
+    it "should stop the sniffer" do
+      # If no sniffer (the default) returns nil
+      expect(subject.sniffer_alive?).to be_falsey
+    end
+
+    it "should wait for in use connections to terminate" do
+      expect(subject).to have_received(:wait_for_in_use_connections).once
+      expect(subject).to have_received(:in_use_connections).twice
+    end
+  end
+
+  describe "connection management" do
+    context "with only one URL in the list" do
+      it "should use the only URL in 'with_connection'" do
+        subject.with_connection do |c|
+          expect(c).to eql(initial_urls.first)
+        end
+      end
+    end
+
+    context "with multiple URLs in the list" do
+      let(:initial_urls) { [ URI.parse("http://localhost:9200"), URI.parse("http://localhost:9201"), URI.parse("http://localhost:9202") ] }
+
+      it "should minimize the number of connections to a single URL" do
+        connected_urls = []
+
+        # If we make 2x the number requests as we have URLs we should
+        # connect to each URL exactly 2 times
+        (initial_urls.size*2).times do
+          u, meta = subject.get_connection
+          connected_urls << u
+        end
+
+        connected_urls.each {|u| subject.return_connection(u) }
+        initial_urls.each do |url|
+          conn_count = connected_urls.select {|u| u == url}.size
+          expect(conn_count).to eql(2)
+        end
+      end
+
+      it "should correctly resurrect the dead" do
+        u,m = subject.get_connection
+
+        # The resurrectionist will call this to check on the backend
+        response = double("response")
+        expect(adapter).to receive(:perform_request).with(u, 'HEAD', subject.healthcheck_path, {}, nil).and_return(response)
+
+        subject.return_connection(u)
+        subject.mark_dead(u, Exception.new)
+
+        expect(subject.url_meta(u)[:dead]).to eql(true)
+        sleep subject.resurrect_delay + 1
+        expect(subject.url_meta(u)[:dead]).to eql(false)
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -19,7 +19,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
     shared_examples("proper host handling") do
       it "should properly transform a host:port string to a URL" do
-        expect(subject.send(:host_to_url, hostname_port)).to eql(http_hostname_port)
+        expect(subject.send(:host_to_url, hostname_port).to_s).to eql(http_hostname_port)
       end
 
       it "should raise an error when a partial URL is an invalid format" do
@@ -29,11 +29,11 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       end
 
       it "should not raise an error with a / for a path" do
-        expect(subject.send(:host_to_url, "#{http_hostname_port}/")).to eql("#{http_hostname_port}/")
+        expect(subject.send(:host_to_url, "#{http_hostname_port}/").to_s).to eql("#{http_hostname_port}/")
       end
 
       it "should parse full URLs correctly" do
-        expect(subject.send(:host_to_url, http_hostname_port)).to eql(http_hostname_port)
+        expect(subject.send(:host_to_url, http_hostname_port).to_s).to eql(http_hostname_port)
       end
 
       it "should reject full URLs with usernames and passwords" do
@@ -56,7 +56,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         end
 
         it "should handle an ssl url correctly when SSL is nil" do
-          expect(subject.send(:host_to_url, https_hostname_port, nil)).to eql(https_hostname_port)
+          expect(subject.send(:host_to_url, https_hostname_port, nil).to_s).to eql(https_hostname_port)
         end
 
         it "should raise an exception if an unexpected value is passed in" do
@@ -66,7 +66,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
       describe "path" do
         it "should allow paths in a url" do
-          expect(subject.send(:host_to_url, http_hostname_port_path, nil)).to eql(http_hostname_port_path)
+          expect(subject.send(:host_to_url, http_hostname_port_path, nil).to_s).to eql(http_hostname_port_path)
         end
 
         it "should not allow paths in two places" do
@@ -76,7 +76,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         end
 
         it "should automatically insert a / in front of path overlays if needed" do
-          expect(subject.send(:host_to_url, http_hostname_port, false, "otherpath")).to eql(http_hostname_port + "/otherpath")
+          expect(subject.send(:host_to_url, http_hostname_port, false, "otherpath")).to eql(URI.parse(http_hostname_port + "/otherpath"))
         end
       end
     end
@@ -99,26 +99,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
   describe "sniffing" do
     let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
-    let(:transport) { client.client.transport }
-
-    before do
-      allow(transport).to receive(:reload_connections!)
-    end
 
     context "with sniffing enabled" do
       let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
 
-      after do
-        client.stop_sniffing!
-      end
-
       it "should start the sniffer" do
-        expect(client.sniffer_thread).to be_a(Thread)
-      end
-
-      it "should periodically sniff the client" do
-        sleep 2
-        expect(transport).to have_received(:reload_connections!).at_least(:once)
+        expect(client.pool.sniffing).to be_truthy
       end
     end
 
@@ -126,7 +112,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       let(:client_opts) { {:sniffing => false} }
 
       it "should not start the sniffer" do
-        expect(client.sniffer_thread).to be_nil
+        expect(client.pool.sniffing).to be_falsey
       end
     end
   end