logstash-output-elasticsearch 1.1.0-java → 2.0.0.beta4-java

data/spec/integration/outputs/templates_spec.rb

@@ -1,97 +1,91 @@
 require_relative "../../../spec/es_spec_helper"
 
 describe "index template expected behavior", :integration => true do
-  ["transport", "http"].each do |protocol|
-    context "with protocol => #{protocol}" do
-        
-      subject! do
-        require "logstash/outputs/elasticsearch"
-        settings = {
-          "manage_template" => true,
-          "template_overwrite" => true,
-          "protocol" => protocol,
-          "host" => "#{get_host()}",
-          "port" => "#{get_port(protocol)}"
-        }
-        next LogStash::Outputs::ElasticSearch.new(settings)
-      end
-
-      before :each do
-        # Delete all templates first.
-        require "elasticsearch"
-
-        # Clean ES of data before we start.
-        @es = get_client
-        @es.indices.delete_template(:name => "*")
-
-        # This can fail if there are no indexes, ignore failure.
-        @es.indices.delete(:index => "*") rescue nil
-
-        subject.register
-
-        subject.receive(LogStash::Event.new("message" => "sample message here"))
-        subject.receive(LogStash::Event.new("somevalue" => 100))
-        subject.receive(LogStash::Event.new("somevalue" => 10))
-        subject.receive(LogStash::Event.new("somevalue" => 1))
-        subject.receive(LogStash::Event.new("country" => "us"))
-        subject.receive(LogStash::Event.new("country" => "at"))
-        subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
-        subject.buffer_flush(:final => true)
-        @es.indices.refresh
-
-        # Wait or fail until everything's indexed.
-        Stud::try(20.times) do
-          r = @es.search
-          insist { r["hits"]["total"] } == 7
-        end
-      end
-
-      it "permits phrase searching on string fields" do
-        results = @es.search(:q => "message:\"sample message\"")
-        insist { results["hits"]["total"] } == 1
-        insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
-      end
-
-      it "numbers dynamically map to a numeric type and permit range queries" do
-        results = @es.search(:q => "somevalue:[5 TO 105]")
-        insist { results["hits"]["total"] } == 2
-
-        values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-        insist { values }.include?(10)
-        insist { values }.include?(100)
-        reject { values }.include?(1)
-      end
-
-      it "does not create .raw field for the message field" do
-        results = @es.search(:q => "message.raw:\"sample message here\"")
-        insist { results["hits"]["total"] } == 0
-      end
-
-      it "creates .raw field from any string field which is not_analyzed" do
-        results = @es.search(:q => "country.raw:\"us\"")
-        insist { results["hits"]["total"] } == 1
-        insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
-
-        # partial or terms should not work.
-        results = @es.search(:q => "country.raw:\"u\"")
-        insist { results["hits"]["total"] } == 0
-      end
-
-      it "make [geoip][location] a geo_point" do
-        results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
-        insist { results["hits"]["total"] } == 1
-        insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
-      end
-
-      it "should index stopwords like 'at' " do
-        results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country" } } } })["aggregations"]["my_agg"]
-        terms = results["buckets"].collect { |b| b["key"] }
-
-        insist { terms }.include?("us")
-
-        # 'at' is a stopword, make sure stopwords are not ignored.
-        insist { terms }.include?("at")
-      end
+  subject! do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "manage_template" => true,
+      "template_overwrite" => true,
+      "hosts" => "#{get_host()}",
+      "port" => "#{get_port()}"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
+
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
+
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+
+    subject.register
+
+    subject.receive(LogStash::Event.new("message" => "sample message here"))
+    subject.receive(LogStash::Event.new("somevalue" => 100))
+    subject.receive(LogStash::Event.new("somevalue" => 10))
+    subject.receive(LogStash::Event.new("somevalue" => 1))
+    subject.receive(LogStash::Event.new("country" => "us"))
+    subject.receive(LogStash::Event.new("country" => "at"))
+    subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
+    subject.buffer_flush(:final => true)
+    @es.indices.refresh
+
+    # Wait or fail until everything's indexed.
+    Stud::try(20.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 7
     end
   end
+
+  it "permits phrase searching on string fields" do
+    results = @es.search(:q => "message:\"sample message\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+  end
+
+  it "numbers dynamically map to a numeric type and permit range queries" do
+    results = @es.search(:q => "somevalue:[5 TO 105]")
+    insist { results["hits"]["total"] } == 2
+
+    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+    insist { values }.include?(10)
+    insist { values }.include?(100)
+    reject { values }.include?(1)
+  end
+
+  it "does not create .raw field for the message field" do
+    results = @es.search(:q => "message.raw:\"sample message here\"")
+    insist { results["hits"]["total"] } == 0
+  end
+
+  it "creates .raw field from any string field which is not_analyzed" do
+    results = @es.search(:q => "country.raw:\"us\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+
+    # partial or terms should not work.
+    results = @es.search(:q => "country.raw:\"u\"")
+    insist { results["hits"]["total"] } == 0
+  end
+
+  it "make [geoip][location] a geo_point" do
+    results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
+  end
+
+  it "should index stopwords like 'at' " do
+    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country" } } } })["aggregations"]["my_agg"]
+    terms = results["buckets"].collect { |b| b["key"] }
+
+    insist { terms }.include?("us")
+
+    # 'at' is a stopword, make sure stopwords are not ignored.
+    insist { terms }.include?("at")
+  end
 end

data/spec/integration/outputs/update_spec.rb

@@ -4,14 +4,13 @@ describe "all protocols update actions", :integration => true do
   require "logstash/outputs/elasticsearch"
   require "elasticsearch"
 
-  def get_es_output( protocol, id = nil, upsert = nil, doc_as_upsert=nil)
+  def get_es_output(id = nil, upsert = nil, doc_as_upsert=nil)
     settings = {
       "manage_template" => true,
       "index" => "logstash-update",
       "template_overwrite" => true,
-      "protocol" => protocol,
-      "host" => get_host(),
-      "port" => get_port(protocol),
+      "hosts" => get_host(),
+      "port" => get_port(),
       "action" => "update"
     }
     settings['upsert'] = upsert unless upsert.nil?
@@ -36,52 +35,48 @@ describe "all protocols update actions", :integration => true do
     @es.indices.refresh
   end
 
-  ["node", "transport", "http"].each do |protocol|
-    context "update only with #{protocol} protocol" do
-      it "should failed without a document_id" do
-        event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
-        action = ["update", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
-        subject = get_es_output(protocol)
-        subject.register
-        expect { subject.flush([action]) }.to raise_error
-      end
+  it "should fail without a document_id" do
+    event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
+    action = ["update", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
+    subject = get_es_output
+    subject.register
+    expect { subject.flush([action]) }.to raise_error
+  end
 
-      it "should not create new document" do
-        subject = get_es_output(protocol, "456")
-        subject.register
-        subject.receive(LogStash::Event.new("message" => "sample message here"))
-        subject.buffer_flush(:final => true)
-        expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-      end
+  it "should not create new document" do
+    subject = get_es_output("456")
+    subject.register
+    subject.receive(LogStash::Event.new("message" => "sample message here"))
+    subject.buffer_flush(:final => true)
+    expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+  end
 
-      it "should update existing document" do
-        subject = get_es_output(protocol, "123")
-        subject.register
-        subject.receive(LogStash::Event.new("message" => "updated message here"))
-        subject.buffer_flush(:final => true)
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-        insist { r["_source"]["message"] } == 'updated message here'
-      end
-    end
+  it "should update existing document" do
+    subject = get_es_output("123")
+    subject.register
+    subject.receive(LogStash::Event.new("message" => "updated message here"))
+    subject.buffer_flush(:final => true)
+    r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+    insist { r["_source"]["message"] } == 'updated message here'
+  end
 
-    context "upsert with #{protocol} protocol" do
-      it "should create new documents with upsert content" do
-        subject = get_es_output(protocol, "456", '{"message": "upsert message"}')
-        subject.register
-        subject.receive(LogStash::Event.new("message" => "sample message here"))
-        subject.buffer_flush(:final => true)
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-        insist { r["_source"]["message"] } == 'upsert message'
-      end
+  context "upsert with protocol" do
+    it "should create new documents with upsert content" do
+      subject = get_es_output("456", '{"message": "upsert message"}')
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'upsert message'
+    end
 
-      it "should create new documents with event/doc as upsert" do
-        subject = get_es_output(protocol, "456", nil, true)
-        subject.register
-        subject.receive(LogStash::Event.new("message" => "sample message here"))
-        subject.buffer_flush(:final => true)
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-        insist { r["_source"]["message"] } == 'sample message here'
-      end
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output("456", nil, true)
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'sample message here'
     end
   end
 end

data/spec/unit/outputs/elasticsearch/protocol_spec.rb

@@ -1,54 +1,63 @@
 require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/elasticsearch/protocol"
+require "logstash/outputs/elasticsearch/http_client"
 require "java"
 
-describe LogStash::Outputs::Elasticsearch::Protocols::NodeClient do
-  context "successful" do
-    it "should map correctly" do
-      index_response = org.elasticsearch.action.index.IndexResponse.new("my_index", "my_type", "my_id", 123, true)
-      update_response = org.elasticsearch.action.update.UpdateResponse.new("my_index", "my_type", "my_id", 123, false)
-      delete_response = org.elasticsearch.action.delete.DeleteResponse.new("my_index", "my_type", "my_id", 123, true)
-      bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", index_response)
-      bulk_item_response_update = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "update", update_response)
-      bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", delete_response)
-      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_update, bulk_item_response_delete], 0)
-      ret = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.normalize_bulk_response(bulk_response)
-      insist { ret } == {"errors" => false}
-    end
-  end
-
-  context "contains failures" do
-    it "should map correctly" do
-      failure = org.elasticsearch.action.bulk.BulkItemResponse::Failure.new("my_index", "my_type", "my_id", "error message", org.elasticsearch.rest.RestStatus::BAD_REQUEST)
-      bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", failure)
-      bulk_item_response_update = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "update", failure)
-      bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", failure)
-      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_update, bulk_item_response_delete], 0)
-      actual = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.normalize_bulk_response(bulk_response)
-      insist { actual } == {"errors" => true, "statuses" => [400, 400, 400]}
-    end
-  end
-end
-
-describe LogStash::Outputs::Elasticsearch::Protocols::HTTPClient do
+describe LogStash::Outputs::Elasticsearch::HttpClient do
   context "successful" do
     it "should map correctly" do
       bulk_response = {"took"=>74, "errors"=>false, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
                                                                           "_type"=>"logs", "_id"=>"AUxTS2C55Jrgi-hC6rQF",
                                                                           "_version"=>1, "status"=>201}}]} 
-      actual = LogStash::Outputs::Elasticsearch::Protocols::HTTPClient.normalize_bulk_response(bulk_response)
+      actual = LogStash::Outputs::Elasticsearch::HttpClient.normalize_bulk_response(bulk_response)
       insist { actual } == {"errors"=> false}
     end
   end
 
   context "contains failures" do
     it "should map correctly" do
+      item_response = {"_index"=>"logstash-2014.11.17",
+                       "_type"=>"logs", "_id"=>"AUxTQ_OI5Jrgi-hC6rQB", "status"=>400,
+                       "error"=>"MapperParsingException[failed to parse]..."}
       bulk_response = {"took"=>71, "errors"=>true,
-                       "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
-                                             "_type"=>"logs", "_id"=>"AUxTQ_OI5Jrgi-hC6rQB", "status"=>400,
-                                             "error"=>"MapperParsingException[failed to parse]..."}}]}
-      actual = LogStash::Outputs::Elasticsearch::Protocols::HTTPClient.normalize_bulk_response(bulk_response)
-      insist { actual } == {"errors"=> true, "statuses"=> [400]}
+                       "items"=>[{"create"=>item_response}]}
+      actual = LogStash::Outputs::Elasticsearch::HttpClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors"=> true, "statuses"=> [400], "details" => [item_response]}
+    end
+  end
+
+  describe "sniffing" do
+    let(:base_options) { {:hosts => ["127.0.0.1"] }}
+    let(:client) { LogStash::Outputs::Elasticsearch::HttpClient.new(base_options.merge(client_opts)) }
+    let(:transport) { client.client.transport }
+
+    before do
+      allow(transport).to receive(:reload_connections!)
+    end
+
+    context "with sniffing enabled" do
+      let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
+
+      after do
+        client.stop_sniffing!
+      end
+
+      it "should start the sniffer" do
+        expect(client.sniffer_thread).to be_a(Thread)
+      end
+
+      it "should periodically sniff the client" do
+        sleep 2
+        expect(transport).to have_received(:reload_connections!)
+      end
+    end
+
+    context "with sniffing disabled" do
+      let(:client_opts) { {:sniffing => false} }
+
+      it "should not start the sniffer" do
+        expect(client.sniffer_thread).to be_nil
+      end
     end
+
   end
 end

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -6,8 +6,7 @@ require "logstash/outputs/elasticsearch"
 describe "Proxy option" do
   let(:settings) {
     {
-      "protocol" => "http",
-      "host" => "node01",
+      "hosts" => "node01",
       "proxy" => proxy
     }
   }
@@ -35,10 +34,10 @@ describe "Proxy option" do
     end
 
     context "when specified as a hash" do
-      let(:proxy) { {"host" => "127.0.0.1", "protocol" => "http"} }
+      let(:proxy) { {"hosts" => "127.0.0.1", "protocol" => "http"} }
 
       it "should pass through the proxy values as symbols" do
-        expected = {:host => proxy["host"], :protocol => proxy["protocol"]}
+        expected = {:hosts => proxy["hosts"], :protocol => proxy["protocol"]}
         expect(::Elasticsearch::Client).to have_received(:new) do |options|
           expect(options[:transport_options][:proxy]).to eql(expected)
         end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1,52 +1,14 @@
 require_relative "../../../spec/es_spec_helper"
-require "flores/random"
 
 describe "outputs/elasticsearch" do
-  context "registration" do
-    it "should register" do
-      output = LogStash::Plugin.lookup("output", "elasticsearch").new("embedded" => "false", "protocol" => "transport", "manage_template" => "false")
-      # register will try to load jars and raise if it cannot find jars
-      expect {output.register}.to_not raise_error
-    end
-
-    it "should fail to register when protocol => http, action => create_unless_exists" do
-      output = LogStash::Plugin.lookup("output", "elasticsearch").new("protocol" => "http", "action" => "create_unless_exists")
-      expect {output.register}.to raise_error
-    end
-  end
-
-  describe "Authentication option" do
-    ["node", "transport"].each do |protocol|
-      context "with protocol => #{protocol}" do
-        subject do
-          require "logstash/outputs/elasticsearch"
-          settings = {
-            "protocol" => protocol,
-            "node_name" => "logstash",
-            "cluster" => "elasticsearch",
-            "host" => "node01",
-            "user" => "test",
-            "password" => "test"
-          }
-          next LogStash::Outputs::ElasticSearch.new(settings)
-        end
-
-        it "should fail in register" do
-          expect {subject.register}.to raise_error
-        end
-      end
-    end
-  end
-
   describe "http client create" do
     require "logstash/outputs/elasticsearch"
     require "elasticsearch"
 
     let(:options) {
       {
-        "protocol" => "http",
         "index" => "my-index",
-        "host" => "localhost",
+        "hosts" => "localhost",
         "path" => "some-path"
       }
     }
@@ -54,7 +16,7 @@ describe "outputs/elasticsearch" do
     let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
 
     let(:manticore_host) {
-      eso.client.first.send(:client).transport.options[:host].first
+      eso.client.send(:client).transport.options[:hosts].first
     }
 
     around(:each) do |block|
@@ -83,117 +45,6 @@ describe "outputs/elasticsearch" do
           expect(manticore_host).to include(options["path"])
         end
       end
-
-
-    end
-  end
-
-  describe "transport protocol" do
-    context "host not configured" do
-      subject do
-        require "logstash/outputs/elasticsearch"
-        settings = {
-          "protocol" => "transport",
-          "node_name" => "mynode"
-        }
-        next LogStash::Outputs::ElasticSearch.new(settings)
-      end
-
-      it "should set host to localhost" do
-        expect(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:new).with({
-          :host => "localhost",
-          :port => "9300-9305",
-          :protocol => "transport",
-          :client_settings => {
-            "client.transport.sniff" => false,
-            "node.name" => "mynode"
-          }
-        })
-        subject.register
-      end
-    end
-
-    context "sniffing => true" do
-      subject do
-        require "logstash/outputs/elasticsearch"
-        settings = {
-          "host" => "node01",
-          "protocol" => "transport",
-          "sniffing" => true
-        }
-        next LogStash::Outputs::ElasticSearch.new(settings)
-      end
-
-      it "should set the sniffing property to true" do
-        expect_any_instance_of(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:client).and_return(nil)
-        subject.register
-        client = subject.instance_eval("@current_client")
-        settings = client.instance_eval("@settings")
-
-        expect(settings.build.getAsMap["client.transport.sniff"]).to eq("true")
-      end
-    end
-
-    context "sniffing => false" do
-      subject do
-        require "logstash/outputs/elasticsearch"
-        settings = {
-          "host" => "node01",
-          "protocol" => "transport",
-          "sniffing" => false
-        }
-        next LogStash::Outputs::ElasticSearch.new(settings)
-      end
-
-      it "should set the sniffing property to true" do
-        expect_any_instance_of(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:client).and_return(nil)
-        subject.register
-        client = subject.instance_eval("@current_client")
-        settings = client.instance_eval("@settings")
-
-        expect(settings.build.getAsMap["client.transport.sniff"]).to eq("false")
-      end
-    end
-  end
-
-
-  # TODO(sissel): Improve this. I'm not a fan of using message expectations (expect().to receive...)
-  # especially with respect to logging to verify a failure/retry has occurred. For now, this 
-  # should suffice, though.
-  context "with timeout set" do
-    let(:listener) { Flores::Random.tcp_listener }
-    let(:port) { listener[2] }
-    let(:options) do
-      {
-        "protocol" => "http",
-        "manage_template" => false,
-        "host" => "localhost",
-        "port" => port,
-        "flush_size" => 1,
-        "timeout" => 1,
-      }
-    end
-    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
-    
-    before do
-      eso.logger = Cabin::Channel.get
-      eso.register 
-
-      # Expect a timeout to be logged.
-      expect(eso.logger).to receive(:warn).with("Failed to flush outgoing items",
-                                                hash_including(:exception => "Manticore::SocketTimeout"))
-    end
-
-    after do
-      listener[0].close
-      eso.close
-    end
-
-    it "should fail after the timeout" do
-      Thread.new { eso.receive(LogStash::Event.new) }
-
-      # Allow the timeout to occur.
-      sleep(options["timeout"] + 0.5)
     end
   end
 end