logstash-output-elasticsearch-leprechaun-fork 1.0.8

data/spec/integration/outputs/elasticsearch/node_spec.rb

@@ -0,0 +1,36 @@
+require_relative "../../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch/protocol"
+
+describe "elasticsearch node client", :integration => true do
+  # Test ElasticSearch Node Client
+  # Reference: http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
+  
+  subject { LogStash::Outputs::Elasticsearch::Protocols::NodeClient }
+
+  it "should support hosts in both string and array" do
+    # Because we defined *hosts* method in NodeClient as private,
+    # we use *obj.send :method,[args...]* to call method *hosts*
+
+    # Node client should support host in string
+    # Case 1: default :host in string
+    insist { subject.send :hosts, :host => "host",:port => 9300 } == "host:9300"
+    # Case 2: :port =~ /^\d+_\d+$/
+    insist { subject.send :hosts, :host => "host",:port => "9300-9302"} == "host:9300,host:9301,host:9302"
+    # Case 3: :host =~ /^.+:.+$/
+    insist { subject.send :hosts, :host => "host:9303",:port => 9300 } == "host:9303"
+    # Case 4:  :host =~ /^.+:.+$/ and :port =~ /^\d+_\d+$/
+    insist { subject.send :hosts, :host => "host:9303",:port => "9300-9302"} == "host:9303"
+
+    # Node client should support host in array
+    # Case 5: :host in array with single item
+    insist { subject.send :hosts, :host => ["host"],:port => 9300 } == ("host:9300")
+    # Case 6: :host in array with more than one items
+    insist { subject.send :hosts, :host => ["host1","host2"],:port => 9300 } == "host1:9300,host2:9300"
+    # Case 7: :host in array with more than one items and :port =~ /^\d+_\d+$/
+    insist { subject.send :hosts, :host => ["host1","host2"],:port => "9300-9302" } == "host1:9300,host1:9301,host1:9302,host2:9300,host2:9301,host2:9302"
+    # Case 8: :host in array with more than one items and some :host =~ /^.+:.+$/
+    insist { subject.send :hosts, :host => ["host1","host2:9303"],:port => 9300 } == "host1:9300,host2:9303"
+    # Case 9: :host in array with more than one items, :port =~ /^\d+_\d+$/ and some :host =~ /^.+:.+$/
+    insist { subject.send :hosts, :host => ["host1","host2:9303"],:port => "9300-9302" } == "host1:9300,host1:9301,host1:9302,host2:9303"
+  end
+end

data/spec/integration/outputs/index_spec.rb

@@ -0,0 +1,90 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "an indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:config) { "not implemented" }
+
+    it "ships events" do
+      insist { config } != "not implemented"
+
+      pipeline = LogStash::Pipeline.new(config)
+      pipeline.run
+
+      index_url = "http://#{get_host}:#{get_port('http')}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("#{index_url}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+
+      response = ftw.get!("#{index_url}/_search?q=*&size=1000")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      result["hits"]["hits"].each do |doc|
+        insist { doc["_type"] } == type
+        insist { doc["_index"] } == index
+      end
+    end
+end
+
+describe "an indexer with custom index_type", :integration => true do
+  it_behaves_like "an indexer" do
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "#{get_host()}"
+          port => "#{get_port('http')}"
+          protocol => "http"
+          index => "#{index}"
+          flush_size => #{flush_size}
+        }
+      }
+      CONFIG
+    }
+  end
+end
+
+describe "an indexer with no type value set (default to logs)", :integration => true do
+  it_behaves_like "an indexer" do
+    let(:type) { "logs" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+        }
+      }
+      output {
+        elasticsearch {
+          host => "#{get_host()}"
+          port => "#{get_port('http')}"
+          protocol => "http"
+          index => "#{index}"
+          flush_size => #{flush_size}
+        }
+      }
+      CONFIG
+    }
+  end
+end

data/spec/integration/outputs/retry_spec.rb

@@ -0,0 +1,156 @@
+require "logstash/outputs/elasticsearch"
+require_relative "../../../spec/es_spec_helper"
+
+describe "failures in bulk class expected behavior", :integration => true do
+  let(:template) { '{"template" : "not important, will be updated by :index"}' }
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:action1) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
+  let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:action2) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event2] }
+  let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+  let(:max_retries) { 3 }
+
+  def mock_actions_with_response(*resp)
+    LogStash::Outputs::Elasticsearch::Protocols::HTTPClient
+      .any_instance.stub(:bulk).and_return(*resp)
+    LogStash::Outputs::Elasticsearch::Protocols::NodeClient
+      .any_instance.stub(:bulk).and_return(*resp)
+  end
+
+  ["transport", "http"].each do |protocol|
+    context "with protocol => #{protocol}" do
+      subject! do
+        settings = {
+          "manage_template" => true,
+          "index" => "logstash-2014.11.17",
+          "template_overwrite" => true,
+          "protocol" => protocol,
+          "host" => get_host(),
+          "port" => get_port(protocol),
+          "retry_max_items" => 10,
+          "retry_max_interval" => 1,
+          "max_retries" => max_retries
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      before :each do
+        # Delete all templates first.
+        require "elasticsearch"
+
+        # Clean ES of data before we start.
+        @es = get_client
+        @es.indices.delete_template(:name => "*")
+        @es.indices.delete(:index => "*")
+        @es.indices.refresh
+      end
+
+      it "should return no errors if all bulk actions are successful" do
+        mock_actions_with_response({"errors" => false})
+        expect(subject).to receive(:submit).with([action1, action2]).once.and_call_original
+        subject.register
+        subject.receive(event1)
+        subject.receive(event2)
+        subject.buffer_flush(:final => true)
+        sleep(2)
+      end
+
+      it "should raise exception and be retried by stud::buffer" do
+        call_count = 0
+        expect(subject).to receive(:submit).with([action1]).exactly(3).times do
+          if (call_count += 1) <= 2
+            raise "error first two times"
+          else
+            {"errors" => false}
+          end
+        end
+        subject.register
+        subject.receive(event1)
+        subject.teardown
+      end
+
+      it "should retry actions with response status of 503" do
+        mock_actions_with_response({"errors" => true, "statuses" => [200, 200, 503, 503]},
+                                   {"errors" => true, "statuses" => [200, 503]},
+                                   {"errors" => false})
+        expect(subject).to receive(:submit).with([action1, action1, action1, action2]).ordered.once.and_call_original
+        expect(subject).to receive(:submit).with([action1, action2]).ordered.once.and_call_original
+        expect(subject).to receive(:submit).with([action2]).ordered.once.and_call_original
+
+        subject.register
+        subject.receive(event1)
+        subject.receive(event1)
+        subject.receive(event1)
+        subject.receive(event2)
+        subject.buffer_flush(:final => true)
+        sleep(3)
+      end
+
+      it "should retry actions with response status of 429" do
+        mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                                   {"errors" => false})
+        expect(subject).to receive(:submit).with([action1]).twice.and_call_original
+        subject.register
+        subject.receive(event1)
+        subject.buffer_flush(:final => true)
+        sleep(3)
+      end
+
+      it "should retry an event until max_retries reached" do
+        mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                                   {"errors" => true, "statuses" => [429]},
+                                   {"errors" => true, "statuses" => [429]},
+                                   {"errors" => true, "statuses" => [429]},
+                                   {"errors" => true, "statuses" => [429]},
+                                   {"errors" => true, "statuses" => [429]})
+        expect(subject).to receive(:submit).with([action1]).exactly(max_retries).times.and_call_original
+        subject.register
+        subject.receive(event1)
+        subject.buffer_flush(:final => true)
+        sleep(3)
+      end
+
+      it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunetly)" do
+        subject.register
+        subject.receive(invalid_event)
+        expect(subject).not_to receive(:retry_push)
+        subject.teardown
+
+        @es.indices.refresh
+        sleep(5)
+        Stud::try(10.times) do
+          r = @es.search
+          insist { r["hits"]["total"] } == 0
+        end
+      end
+
+      it "successful requests should not be appended to retry queue" do
+        subject.register
+        subject.receive(event1)
+        expect(subject).not_to receive(:retry_push)
+        subject.teardown
+
+        @es.indices.refresh
+        sleep(5)
+        Stud::try(10.times) do
+          r = @es.search
+          insist { r["hits"]["total"] } == 1
+        end
+      end
+
+      it "should only index proper events" do
+        subject.register
+        subject.receive(invalid_event)
+        subject.receive(event1)
+        subject.teardown
+
+        @es.indices.refresh
+        sleep(5)
+        Stud::try(10.times) do
+          r = @es.search
+          insist { r["hits"]["total"] } == 1
+        end
+      end
+    end
+  end
+end

data/spec/integration/outputs/routing_spec.rb

@@ -0,0 +1,114 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "a routing indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:routing) { "not_implemented" }
+    let(:config) { "not_implemented" }
+
+    it "ships events" do
+      insist { routing } != "not_implemented"
+      insist { config } != "not_implemented"
+
+      pipeline = LogStash::Pipeline.new(config)
+      pipeline.run
+
+      index_url = "http://#{get_host()}:#{get_port('http')}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("#{index_url}/_count?q=*&routing=#{routing}")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+    end
+end
+
+describe "(http protocol) index events with static routing", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "#{get_host()}"
+          port => "#{get_port('http')}"
+          protocol => "http"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "#{routing}"
+        }
+      }
+      CONFIG
+    }
+  end
+end
+
+describe "(http_protocol) index events with fieldref in routing value", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "#{routing}"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "#{get_host()}"
+          port => "#{get_port('http')}"
+          protocol => "http"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "%{message}"
+        }
+      }
+      CONFIG
+    }
+  end
+end
+
+describe "(transport protocol) index events with fieldref in routing value", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "#{routing}"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "#{get_host()}"
+          port => "#{get_port('transport')}"
+          protocol => "transport"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "%{message}"
+        }
+      }
+      CONFIG
+    }
+  end
+end

data/spec/integration/outputs/secure_spec.rb

@@ -0,0 +1,113 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "protocol" => "http",
+      "node_name" => "logstash",
+      "cluster" => "elasticsearch",
+      "host" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+      "ssl" => true,
+      "cacert" => "/tmp/ca/certs/cacert.pem",
+      # or
+      #"truststore" => "/tmp/ca/truststore.jks",
+      #"truststore_password" => "testeteste"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "protocol" => "http",
+      "cluster" => "elasticsearch",
+      "host" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "protocol" => "http",
+      "node_name" => "logstash",
+      "cluster" => "elasticsearch",
+      "host" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+      "ssl" => true,
+      "cacert" => "/tmp/ca/certs/cacert.pem",
+      # or
+      #"truststore" => "/tmp/ca/truststore.jks",
+      #"truststore_password" => "testeteste"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "protocol" => "http",
+      "cluster" => "elasticsearch",
+      "host" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end