logstash-output-elasticsearch 1.1.0-java → 2.0.0.beta4-java

data/spec/es_spec_helper.rb

@@ -9,57 +9,61 @@ CONTAINER_NAME = "logstash-output-elasticsearch-#{rand(999).to_s}"
 CONTAINER_IMAGE = "elasticsearch"
 CONTAINER_TAG = "1.6"
 
-module ESHelper
+DOCKER_INTEGRATION = ENV["DOCKER_INTEGRATION"]
 
+module ESHelper
   def get_host
-    Longshoreman.new.get_host_ip
+    DOCKER_INTEGRATION ? Longshoreman.new.get_host_ip : "127.0.0.1"
   end
 
-  def get_port(protocol)
+  def get_port
+    return 9200 unless DOCKER_INTEGRATION
+
     container = Longshoreman::Container.new
     container.get(CONTAINER_NAME)
-    case protocol
-    when "http"
-      container.rport(9200)
-    when "transport", "node"
-      container.rport(9300)
-    end
+    container.rport(9200)
   end
 
   def get_client
-    Elasticsearch::Client.new(:host => "#{get_host}:#{get_port('http')}")
+    Elasticsearch::Client.new(:host => "#{get_host}:#{get_port}")
   end
 end
 
+
 RSpec.configure do |config|
   config.include ESHelper
 
-  # this :all hook gets run before every describe block that is tagged with :integration => true.
-  config.before(:all, :integration => true) do
-    # check if container exists already before creating new one.
-    begin
-      ls = Longshoreman::new
-      ls.container.get(CONTAINER_NAME)
-    rescue Docker::Error::NotFoundError
-      Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME)
-      # TODO(talevy): verify ES is running instead of static timeout
-      sleep 10
+
+  if DOCKER_INTEGRATION
+    # this :all hook gets run before every describe block that is tagged with :integration => true.
+    config.before(:all, :integration => true) do
+
+
+      # check if container exists already before creating new one.
+      begin
+        ls = Longshoreman::new
+        ls.container.get(CONTAINER_NAME)
+      rescue Docker::Error::NotFoundError
+        Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME)
+        # TODO(talevy): verify ES is running instead of static timeout
+        sleep 10
+      end
     end
-  end
 
-  # we want to do a final cleanup after all :integration runs,
-  # but we don't want to clean up before the last block.
-  # This is a final blind check to see if the ES docker container is running and
-  # needs to be cleaned up. If no container can be found and/or docker is not
-  # running on the system, we do nothing.
-  config.after(:suite) do
-    # only cleanup docker container if system has docker and the container is running
-    begin
-      ls = Longshoreman::new
-      ls.container.get(CONTAINER_NAME)
-      ls.cleanup
-    rescue Docker::Error::NotFoundError, Excon::Errors::SocketError
-      # do nothing
+    # we want to do a final cleanup after all :integration runs,
+    # but we don't want to clean up before the last block.
+    # This is a final blind check to see if the ES docker container is running and
+    # needs to be cleaned up. If no container can be found and/or docker is not
+    # running on the system, we do nothing.
+    config.after(:suite) do
+      # only cleanup docker container if system has docker and the container is running
+      begin
+        ls = Longshoreman::new
+        ls.container.get(CONTAINER_NAME)
+        ls.cleanup
+      rescue Docker::Error::NotFoundError, Excon::Errors::SocketError
+        # do nothing
+      end
     end
   end
 end

data/spec/integration/outputs/create_spec.rb

@@ -0,0 +1,56 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "client create actions", :integration => true do
+  require "logstash/outputs/elasticsearch"
+  require "elasticsearch"
+
+  def get_es_output(action, id = nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-create",
+      "template_overwrite" => true,
+      "hosts" => get_host(),
+      "port" => get_port(),
+      "action" => action
+    }
+    settings['document_id'] = id unless id.nil?
+    LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  context "when action => create" do
+    it "should create new documents with or without id" do
+      subject = get_es_output("create", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+
+    it "should create new documents without id" do
+      subject = get_es_output("create")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+  end
+end

data/spec/integration/outputs/index_spec.rb

@@ -13,7 +13,7 @@ shared_examples "an indexer" do
       pipeline = LogStash::Pipeline.new(config)
       pipeline.run
 
-      index_url = "http://#{get_host}:#{get_port('http')}/#{index}"
+      index_url = "http://#{get_host}:#{get_port}/#{index}"
 
       ftw = FTW::Agent.new
       ftw.post!("#{index_url}/_refresh")
@@ -52,9 +52,8 @@ describe "an indexer with custom index_type", :integration => true do
       }
       output {
         elasticsearch {
-          host => "#{get_host()}"
-          port => "#{get_port('http')}"
-          protocol => "http"
+          hosts => "#{get_host()}"
+          port => "#{get_port}"
           index => "#{index}"
           flush_size => #{flush_size}
         }
@@ -77,9 +76,8 @@ describe "an indexer with no type value set (default to logs)", :integration =>
       }
       output {
         elasticsearch {
-          host => "#{get_host()}"
-          port => "#{get_port('http')}"
-          protocol => "http"
+          hosts => "#{get_host()}"
+          port => "#{get_port}"
           index => "#{index}"
           flush_size => #{flush_size}
         }

data/spec/integration/outputs/retry_spec.rb

@@ -11,146 +11,138 @@ describe "failures in bulk class expected behavior", :integration => true do
   let(:max_retries) { 3 }
 
   def mock_actions_with_response(*resp)
-    LogStash::Outputs::Elasticsearch::Protocols::HTTPClient
-      .any_instance.stub(:bulk).and_return(*resp)
-    LogStash::Outputs::Elasticsearch::Protocols::NodeClient
+    LogStash::Outputs::Elasticsearch::HttpClient
       .any_instance.stub(:bulk).and_return(*resp)
   end
 
-  ["transport", "http"].each do |protocol|
-    context "with protocol => #{protocol}" do
-      subject! do
-        settings = {
-          "manage_template" => true,
-          "index" => "logstash-2014.11.17",
-          "template_overwrite" => true,
-          "protocol" => protocol,
-          "host" => get_host(),
-          "port" => get_port(protocol),
-          "retry_max_items" => 10,
-          "retry_max_interval" => 1,
-          "max_retries" => max_retries
-        }
-        next LogStash::Outputs::ElasticSearch.new(settings)
-      end
+  subject! do
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-2014.11.17",
+      "template_overwrite" => true,
+      "hosts" => get_host(),
+      "port" => get_port(),
+      "retry_max_items" => 10,
+      "retry_max_interval" => 1,
+      "max_retries" => max_retries
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
 
-      before :each do
-        # Delete all templates first.
-        require "elasticsearch"
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
 
-        # Clean ES of data before we start.
-        @es = get_client
-        @es.indices.delete_template(:name => "*")
-        @es.indices.delete(:index => "*")
-        @es.indices.refresh
-      end
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
+    @es.indices.delete(:index => "*")
+    @es.indices.refresh
+  end
 
-      it "should return no errors if all bulk actions are successful" do
-        mock_actions_with_response({"errors" => false})
-        expect(subject).to receive(:submit).with([action1, action2]).once.and_call_original
-        subject.register
-        subject.receive(event1)
-        subject.receive(event2)
-        subject.buffer_flush(:final => true)
-        sleep(2)
-      end
+  it "should return no errors if all bulk actions are successful" do
+    mock_actions_with_response({"errors" => false})
+    expect(subject).to receive(:submit).with([action1, action2]).once.and_call_original
+    subject.register
+    subject.receive(event1)
+    subject.receive(event2)
+    subject.buffer_flush(:final => true)
+    sleep(2)
+  end
 
-      it "should raise exception and be retried by stud::buffer" do
-        call_count = 0
-        expect(subject).to receive(:submit).with([action1]).exactly(3).times do
-          if (call_count += 1) <= 2
-            raise "error first two times"
-          else
-            {"errors" => false}
-          end
-        end
-        subject.register
-        subject.receive(event1)
-        subject.teardown
+  it "should raise exception and be retried by stud::buffer" do
+    call_count = 0
+    expect(subject).to receive(:submit).with([action1]).exactly(3).times do
+      if (call_count += 1) <= 2
+        raise "error first two times"
+      else
+        {"errors" => false}
       end
+    end
+    subject.register
+    subject.receive(event1)
+    subject.teardown
+  end
 
-      it "should retry actions with response status of 503" do
-        mock_actions_with_response({"errors" => true, "statuses" => [200, 200, 503, 503]},
-                                   {"errors" => true, "statuses" => [200, 503]},
-                                   {"errors" => false})
-        expect(subject).to receive(:submit).with([action1, action1, action1, action2]).ordered.once.and_call_original
-        expect(subject).to receive(:submit).with([action1, action2]).ordered.once.and_call_original
-        expect(subject).to receive(:submit).with([action2]).ordered.once.and_call_original
-
-        subject.register
-        subject.receive(event1)
-        subject.receive(event1)
-        subject.receive(event1)
-        subject.receive(event2)
-        subject.buffer_flush(:final => true)
-        sleep(3)
-      end
+  it "should retry actions with response status of 503" do
+    mock_actions_with_response({"errors" => true, "statuses" => [200, 200, 503, 503]},
+                               {"errors" => true, "statuses" => [200, 503]},
+                               {"errors" => false})
+    expect(subject).to receive(:submit).with([action1, action1, action1, action2]).ordered.once.and_call_original
+    expect(subject).to receive(:submit).with([action1, action2]).ordered.once.and_call_original
+    expect(subject).to receive(:submit).with([action2]).ordered.once.and_call_original
+
+    subject.register
+    subject.receive(event1)
+    subject.receive(event1)
+    subject.receive(event1)
+    subject.receive(event2)
+    subject.buffer_flush(:final => true)
+    sleep(3)
+  end
 
-      it "should retry actions with response status of 429" do
-        mock_actions_with_response({"errors" => true, "statuses" => [429]},
-                                   {"errors" => false})
-        expect(subject).to receive(:submit).with([action1]).twice.and_call_original
-        subject.register
-        subject.receive(event1)
-        subject.buffer_flush(:final => true)
-        sleep(3)
-      end
+  it "should retry actions with response status of 429" do
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => false})
+    expect(subject).to receive(:submit).with([action1]).twice.and_call_original
+    subject.register
+    subject.receive(event1)
+    subject.buffer_flush(:final => true)
+    sleep(3)
+  end
 
-      it "should retry an event until max_retries reached" do
-        mock_actions_with_response({"errors" => true, "statuses" => [429]},
-                                   {"errors" => true, "statuses" => [429]},
-                                   {"errors" => true, "statuses" => [429]},
-                                   {"errors" => true, "statuses" => [429]},
-                                   {"errors" => true, "statuses" => [429]},
-                                   {"errors" => true, "statuses" => [429]})
-        expect(subject).to receive(:submit).with([action1]).exactly(max_retries).times.and_call_original
-        subject.register
-        subject.receive(event1)
-        subject.buffer_flush(:final => true)
-        sleep(3)
-      end
+  it "should retry an event until max_retries reached" do
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]})
+    expect(subject).to receive(:submit).with([action1]).exactly(max_retries).times.and_call_original
+    subject.register
+    subject.receive(event1)
+    subject.buffer_flush(:final => true)
+    sleep(3)
+  end
 
-      it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunetly)" do
-        subject.register
-        subject.receive(invalid_event)
-        expect(subject).not_to receive(:retry_push)
-        subject.teardown
-
-        @es.indices.refresh
-        sleep(5)
-        Stud::try(10.times) do
-          r = @es.search
-          insist { r["hits"]["total"] } == 0
-        end
-      end
+  it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunately)" do
+    subject.register
+    subject.receive(invalid_event)
+    expect(subject).not_to receive(:retry_push)
+    subject.teardown
+
+    @es.indices.refresh
+    sleep(5)
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 0
+    end
+  end
 
-      it "successful requests should not be appended to retry queue" do
-        subject.register
-        subject.receive(event1)
-        expect(subject).not_to receive(:retry_push)
-        subject.teardown
-
-        @es.indices.refresh
-        sleep(5)
-        Stud::try(10.times) do
-          r = @es.search
-          insist { r["hits"]["total"] } == 1
-        end
-      end
+  it "successful requests should not be appended to retry queue" do
+    subject.register
+    subject.receive(event1)
+    expect(subject).not_to receive(:retry_push)
+    subject.teardown
+    @es.indices.refresh
+    sleep(5)
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 1
+    end
+  end
 
-      it "should only index proper events" do
-        subject.register
-        subject.receive(invalid_event)
-        subject.receive(event1)
-        subject.teardown
-
-        @es.indices.refresh
-        sleep(5)
-        Stud::try(10.times) do
-          r = @es.search
-          insist { r["hits"]["total"] } == 1
-        end
-      end
+  it "should only index proper events" do
+    subject.register
+    subject.receive(invalid_event)
+    subject.receive(event1)
+    subject.teardown
+
+    @es.indices.refresh
+    sleep(5)
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 1
     end
   end
 end

data/spec/integration/outputs/routing_spec.rb

@@ -15,7 +15,7 @@ shared_examples "a routing indexer" do
       pipeline = LogStash::Pipeline.new(config)
       pipeline.run
 
-      index_url = "http://#{get_host()}:#{get_port('http')}/#{index}"
+      index_url = "http://#{get_host()}:#{get_port()}/#{index}"
 
       ftw = FTW::Agent.new
       ftw.post!("#{index_url}/_refresh")
@@ -46,9 +46,8 @@ describe "(http protocol) index events with static routing", :integration => tru
       }
       output {
         elasticsearch {
-          host => "#{get_host()}"
-          port => "#{get_port('http')}"
-          protocol => "http"
+          hosts => "#{get_host()}"
+          port => "#{get_port()}"
           index => "#{index}"
           flush_size => #{flush_size}
           routing => "#{routing}"
@@ -73,9 +72,8 @@ describe "(http_protocol) index events with fieldref in routing value", :integra
       }
       output {
         elasticsearch {
-          host => "#{get_host()}"
-          port => "#{get_port('http')}"
-          protocol => "http"
+          hosts => "#{get_host()}"
+          port => "#{get_port()}"
           index => "#{index}"
           flush_size => #{flush_size}
           routing => "%{message}"
@@ -86,29 +84,3 @@ describe "(http_protocol) index events with fieldref in routing value", :integra
   end
 end
 
-describe "(transport protocol) index events with fieldref in routing value", :integration => true do
-  it_behaves_like 'a routing indexer' do
-    let(:routing) { "test" }
-    let(:config) {
-      <<-CONFIG
-      input {
-        generator {
-          message => "#{routing}"
-          count => #{event_count}
-          type => "#{type}"
-        }
-      }
-      output {
-        elasticsearch {
-          host => "#{get_host()}"
-          port => "#{get_port('transport')}"
-          protocol => "transport"
-          index => "#{index}"
-          flush_size => #{flush_size}
-          routing => "%{message}"
-        }
-      }
-      CONFIG
-    }
-  end
-end

data/spec/integration/outputs/secure_spec.rb

@@ -4,10 +4,9 @@ describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure =>
   subject do
     require "logstash/outputs/elasticsearch"
     settings = {
-      "protocol" => "http",
       "node_name" => "logstash",
       "cluster" => "elasticsearch",
-      "host" => "node01",
+      "hosts" => "node01",
       "user" => "user",
       "password" => "changeme",
       "ssl" => true,
@@ -35,9 +34,8 @@ describe "connect using HTTP Authentication", :elasticsearch_secure => true do
   subject do
     require "logstash/outputs/elasticsearch"
     settings = {
-      "protocol" => "http",
       "cluster" => "elasticsearch",
-      "host" => "node01",
+      "hosts" => "node01",
       "user" => "user",
       "password" => "changeme",
     }
@@ -60,10 +58,9 @@ describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure =>
   subject do
     require "logstash/outputs/elasticsearch"
     settings = {
-      "protocol" => "http",
       "node_name" => "logstash",
       "cluster" => "elasticsearch",
-      "host" => "node01",
+      "hosts" => "node01",
       "user" => "user",
       "password" => "changeme",
       "ssl" => true,
@@ -91,9 +88,7 @@ describe "connect using HTTP Authentication", :elasticsearch_secure => true do
   subject do
     require "logstash/outputs/elasticsearch"
     settings = {
-      "protocol" => "http",
-      "cluster" => "elasticsearch",
-      "host" => "node01",
+      "hosts" => "node01",
       "user" => "user",
       "password" => "changeme",
     }