logstash-output-elasticsearch_java 1.0.0.beta1

data/spec/integration/outputs/elasticsearch/node_spec.rb

@@ -0,0 +1,36 @@
+require_relative "../../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch_java/protocol"
+
+describe "elasticsearch node client", :integration => true do
+  # Test ElasticSearch Node Client
+  # Reference: http://www.elasticsearch.org/guide/reference/modules/discovery/zen/
+  
+  subject { LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient }
+
+  it "should support hosts in both string and array" do
+    # Because we defined *hosts* method in NodeClient as private,
+    # we use *obj.send :method,[args...]* to call method *hosts*
+
+    # Node client should support host in string
+    # Case 1: default :hosts in string
+    insist { subject.send :hosts, :hosts => "host",:port => 9300 } == "host:9300"
+    # Case 2: :port =~ /^\d+_\d+$/
+    insist { subject.send :hosts, :hosts => "host",:port => "9300-9302"} == "host:9300,host:9301,host:9302"
+    # Case 3: :hosts =~ /^.+:.+$/
+    insist { subject.send :hosts, :hosts=> "host:9303",:port => 9300 } == "host:9303"
+    # Case 4:  :hosts=~ /^.+:.+$/ and :port =~ /^\d+_\d+$/
+    insist { subject.send :hosts, :hosts => "host:9303",:port => "9300-9302"} == "host:9303"
+
+    # Node client should support host in array
+    # Case 5: :hosts in array with single item
+    insist { subject.send :hosts, :hosts => ["host"],:port => 9300 } == ("host:9300")
+    # Case 6: :hostsin array with more than one items
+    insist { subject.send :hosts, :hosts=> ["host1","host2"],:port => 9300 } == "host1:9300,host2:9300"
+    # Case 7: :hostsin array with more than one items and :port =~ /^\d+_\d+$/
+    insist { subject.send :hosts, :hosts=> ["host1","host2"],:port => "9300-9302" } == "host1:9300,host1:9301,host1:9302,host2:9300,host2:9301,host2:9302"
+    # Case 8: :hostsin array with more than one items and some :hosts=~ /^.+:.+$/
+    insist { subject.send :hosts, :hosts=> ["host1","host2:9303"],:port => 9300 } == "host1:9300,host2:9303"
+    # Case 9: :hostsin array with more than one items, :port =~ /^\d+_\d+$/ and some :hosts=~ /^.+:.+$/
+    insist { subject.send :hosts, :hosts => ["host1","host2:9303"],:port => "9300-9302" } == "host1:9300,host1:9301,host1:9302,host2:9303"
+  end
+end

data/spec/integration/outputs/index_spec.rb

@@ -0,0 +1,90 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "an indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:config) { "not implemented" }
+
+    it "ships events" do
+      insist { config } != "not implemented"
+
+      pipeline = LogStash::Pipeline.new(config)
+      pipeline.run
+
+      index_url = "http://#{get_host}:#{get_port('http')}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("#{index_url}/_count?q=*")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+
+      response = ftw.get!("#{index_url}/_search?q=*&size=1000")
+      data = ""
+      response.read_body { |chunk| data << chunk }
+      result = LogStash::Json.load(data)
+      result["hits"]["hits"].each do |doc|
+        insist { doc["_type"] } == type
+        insist { doc["_index"] } == index
+      end
+    end
+end
+
+describe "an indexer with custom index_type", :integration => true do
+  it_behaves_like "an indexer" do
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch_java {
+          hosts => "#{get_host()}"
+          port => "#{get_port('transport')}"
+          protocol => "transport"
+          index => "#{index}"
+          flush_size => #{flush_size}
+        }
+      }
+      CONFIG
+    }
+  end
+end
+
+describe "an indexer with no type value set (default to logs)", :integration => true do
+  it_behaves_like "an indexer" do
+    let(:type) { "logs" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+        }
+      }
+      output {
+        elasticsearch_java {
+          hosts => "#{get_host()}"
+          port => "#{get_port('transport')}"
+          protocol => "transport"
+          index => "#{index}"
+          flush_size => #{flush_size}
+        }
+      }
+      CONFIG
+    }
+  end
+end

data/spec/integration/outputs/retry_spec.rb

@@ -0,0 +1,148 @@
+require "elasticsearch"
+require "logstash/outputs/elasticsearch_java"
+require_relative "../../../spec/es_spec_helper"
+
+describe "failures in bulk class expected behavior", :integration => true do
+  let(:template) { '{"template" : "not important, will be updated by :index"}' }
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:action1) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
+  let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:action2) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event2] }
+  let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+  let(:max_retries) { 3 }
+
+  def mock_actions_with_response(*resp)
+    allow_any_instance_of(LogStash::Outputs::ElasticSearchJavaPlugins::Protocols::NodeClient).to receive(:bulk).and_return(*resp)
+  end
+
+
+  subject! do
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-2014.11.17",
+      "template_overwrite" => true,
+      "protocol" => 'transport',
+      "hosts" => get_host(),
+      "port" => get_port('transport'),
+      "retry_max_items" => 10,
+      "retry_max_interval" => 1,
+      "max_retries" => max_retries
+    }
+    next LogStash::Outputs::ElasticSearchJava.new(settings)
+  end
+
+  before :each do
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
+    @es.indices.delete(:index => "*")
+    @es.indices.refresh
+  end
+
+  it "should return no errors if all bulk actions are successful" do
+    mock_actions_with_response({"errors" => false})
+    expect(subject).to receive(:submit).with([action1, action2]).once.and_call_original
+    subject.register
+    subject.receive(event1)
+    subject.receive(event2)
+    subject.buffer_flush(:final => true)
+    sleep(2)
+  end
+
+  it "should raise exception and be retried by stud::buffer" do
+    call_count = 0
+    expect(subject).to receive(:submit).with([action1]).exactly(3).times do
+      if (call_count += 1) <= 2
+        raise "error first two times"
+      else
+        {"errors" => false}
+      end
+    end
+    subject.register
+    subject.receive(event1)
+    subject.teardown
+  end
+
+  it "should retry actions with response status of 503" do
+    mock_actions_with_response({"errors" => true, "statuses" => [200, 200, 503, 503]},
+                               {"errors" => true, "statuses" => [200, 503]},
+                               {"errors" => false})
+    expect(subject).to receive(:submit).with([action1, action1, action1, action2]).ordered.once.and_call_original
+    expect(subject).to receive(:submit).with([action1, action2]).ordered.once.and_call_original
+    expect(subject).to receive(:submit).with([action2]).ordered.once.and_call_original
+
+    subject.register
+    subject.receive(event1)
+    subject.receive(event1)
+    subject.receive(event1)
+    subject.receive(event2)
+    subject.buffer_flush(:final => true)
+    sleep(3)
+  end
+
+  it "should retry actions with response status of 429" do
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => false})
+    expect(subject).to receive(:submit).with([action1]).twice.and_call_original
+    subject.register
+    subject.receive(event1)
+    subject.buffer_flush(:final => true)
+    sleep(3)
+  end
+
+  it "should retry an event until max_retries reached" do
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]})
+    expect(subject).to receive(:submit).with([action1]).exactly(max_retries).times.and_call_original
+    subject.register
+    subject.receive(event1)
+    subject.buffer_flush(:final => true)
+    sleep(3)
+  end
+
+  it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunetly)" do
+    subject.register
+    subject.receive(invalid_event)
+    expect(subject).not_to receive(:retry_push)
+    subject.teardown
+
+    @es.indices.refresh
+    sleep(5)
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 0
+    end
+  end
+
+  it "successful requests should not be appended to retry queue" do
+    subject.register
+    subject.receive(event1)
+    expect(subject).not_to receive(:retry_push)
+    subject.teardown
+
+    @es.indices.refresh
+    sleep(5)
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 1
+    end
+  end
+
+  it "should only index proper events" do
+    subject.register
+    subject.receive(invalid_event)
+    subject.receive(event1)
+    subject.teardown
+    @es.indices.refresh
+    sleep(5)
+
+    Stud::try(10.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 1
+    end
+  end
+end

data/spec/integration/outputs/routing_spec.rb

@@ -0,0 +1,60 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "a routing indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:routing) { "not_implemented" }
+    let(:config) { "not_implemented" }
+
+    it "ships events" do
+      insist { routing } != "not_implemented"
+      insist { config } != "not_implemented"
+
+      pipeline = LogStash::Pipeline.new(config)
+      pipeline.run
+
+      index_url = "http://#{get_host()}:#{get_port('http')}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("#{index_url}/_count?q=*&routing=#{routing}")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+    end
+end
+
+describe "(transport protocol) index events with fieldref in routing value", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "#{routing}"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch_java {
+          hosts => "#{get_host()}"
+          port => "#{get_port('transport')}"
+          protocol => "transport"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "%{message}"
+        }
+      }
+      CONFIG
+    }
+  end
+end

data/spec/integration/outputs/secure_spec.rb

@@ -0,0 +1,113 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch_java"
+    settings = {
+      "protocol" => "http",
+      "node_name" => "logstash",
+      "cluster" => "elasticsearch",
+      "hosts" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+      "ssl" => true,
+      "cacert" => "/tmp/ca/certs/cacert.pem",
+      # or
+      #"truststore" => "/tmp/ca/truststore.jks",
+      #"truststore_password" => "testeteste"
+    }
+    next LogStash::Outputs::ElasticSearchJava.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch_java"
+    settings = {
+      "protocol" => "http",
+      "cluster" => "elasticsearch",
+      "hosts" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+    }
+    next LogStash::Outputs::ElasticSearchJava.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch_java"
+    settings = {
+      "protocol" => "http",
+      "node_name" => "logstash",
+      "cluster" => "elasticsearch",
+      "hosts" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+      "ssl" => true,
+      "cacert" => "/tmp/ca/certs/cacert.pem",
+      # or
+      #"truststore" => "/tmp/ca/truststore.jks",
+      #"truststore_password" => "testeteste"
+    }
+    next LogStash::Outputs::ElasticSearchJava.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch_java"
+    settings = {
+      "protocol" => "http",
+      "cluster" => "elasticsearch",
+      "hosts" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+    }
+    next LogStash::Outputs::ElasticSearchJava.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end

data/spec/integration/outputs/templates_spec.rb

@@ -0,0 +1,97 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "index template expected behavior", :integration => true do
+  ["transport"].each do |protocol|
+    context "with protocol => #{protocol}" do
+        
+      subject! do
+        require "logstash/outputs/elasticsearch_java"
+        settings = {
+          "manage_template" => true,
+          "template_overwrite" => true,
+          "protocol" => protocol,
+          "hosts" => "#{get_host()}",
+          "port" => "#{get_port('transport')}"
+        }
+        next LogStash::Outputs::ElasticSearchJava.new(settings)
+      end
+
+      before :each do
+        # Delete all templates first.
+        require "elasticsearch"
+
+        # Clean ES of data before we start.
+        @es = get_client
+        @es.indices.delete_template(:name => "*")
+
+        # This can fail if there are no indexes, ignore failure.
+        @es.indices.delete(:index => "*") rescue nil
+
+        subject.register
+
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.receive(LogStash::Event.new("somevalue" => 100))
+        subject.receive(LogStash::Event.new("somevalue" => 10))
+        subject.receive(LogStash::Event.new("somevalue" => 1))
+        subject.receive(LogStash::Event.new("country" => "us"))
+        subject.receive(LogStash::Event.new("country" => "at"))
+        subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
+        subject.buffer_flush(:final => true)
+        @es.indices.refresh
+
+        # Wait or fail until everything's indexed.
+        Stud::try(20.times) do
+          r = @es.search
+          insist { r["hits"]["total"] } == 7
+        end
+      end
+
+      it "permits phrase searching on string fields" do
+        results = @es.search(:q => "message:\"sample message\"")
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+      end
+
+      it "numbers dynamically map to a numeric type and permit range queries" do
+        results = @es.search(:q => "somevalue:[5 TO 105]")
+        insist { results["hits"]["total"] } == 2
+
+        values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+        insist { values }.include?(10)
+        insist { values }.include?(100)
+        reject { values }.include?(1)
+      end
+
+      it "does not create .raw field for the message field" do
+        results = @es.search(:q => "message.raw:\"sample message here\"")
+        insist { results["hits"]["total"] } == 0
+      end
+
+      it "creates .raw field from any string field which is not_analyzed" do
+        results = @es.search(:q => "country.raw:\"us\"")
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+
+        # partial or terms should not work.
+        results = @es.search(:q => "country.raw:\"u\"")
+        insist { results["hits"]["total"] } == 0
+      end
+
+      it "make [geoip][location] a geo_point" do
+        results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
+      end
+
+      it "should index stopwords like 'at' " do
+        results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country" } } } })["aggregations"]["my_agg"]
+        terms = results["buckets"].collect { |b| b["key"] }
+
+        insist { terms }.include?("us")
+
+        # 'at' is a stopword, make sure stopwords are not ignored.
+        insist { terms }.include?("at")
+      end
+    end
+  end
+end

data/spec/integration/outputs/transport_create_spec.rb

@@ -0,0 +1,94 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "transport client create actions", :integration => true do
+  require "logstash/outputs/elasticsearch_java"
+  require "elasticsearch"
+
+  def get_es_output(action, id = nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-create",
+      "template_overwrite" => true,
+      "protocol" => "transport",
+      "hosts" => get_host(),
+      "port" => get_port('transport'),
+      "action" => action
+    }
+    settings['document_id'] = id unless id.nil?
+    LogStash::Outputs::ElasticSearchJava.new(settings)
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  context "when action => create" do
+    it "should create new documents with or without id" do
+      subject = get_es_output("create", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+
+    it "should create new documents without id" do
+      subject = get_es_output("create")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+  end
+
+  context "when action => create_unless_exists" do
+    it "should create new documents when specific id is specified" do
+      subject = get_es_output("create_unless_exists", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+
+    it "should fail to create a document when no id is specified" do
+      event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
+      action = ["create_unless_exists", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
+      subject = get_es_output(action[0])
+      subject.register
+      expect { subject.flush([action]) }.to raise_error
+    end
+
+    it "should unsuccesfully submit two records with the same document id" do
+      subject = get_es_output("create_unless_exists", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.receive(LogStash::Event.new("message" => "sample message here")) # 400 status failure (same id)
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+  end
+end