logstash-output-elasticsearch 0.2.8-java → 0.2.9-java

data/spec/integration/outputs/routing_spec.rb

@@ -0,0 +1,114 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "a routing indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:flush_size) { rand(200) + 1 }
+    let(:routing) { "not_implemented" }
+    let(:config) { "not_implemented" }
+
+    it "ships events" do
+      insist { routing } != "not_implemented"
+      insist { config } != "not_implemented"
+
+      pipeline = LogStash::Pipeline.new(config)
+      pipeline.run
+
+      index_url = "http://#{get_host()}:#{get_port('http')}/#{index}"
+
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+        response = ftw.get!("#{index_url}/_count?q=*&routing=#{routing}")
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+    end
+end
+
+describe "(http protocol) index events with static routing", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "#{get_host()}"
+          port => "#{get_port('http')}"
+          protocol => "http"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "#{routing}"
+        }
+      }
+      CONFIG
+    }
+  end
+end
+
+describe "(http_protocol) index events with fieldref in routing value", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "#{routing}"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "#{get_host()}"
+          port => "#{get_port('http')}"
+          protocol => "http"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "%{message}"
+        }
+      }
+      CONFIG
+    }
+  end
+end
+
+describe "(transport protocol) index events with fieldref in routing value", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      <<-CONFIG
+      input {
+        generator {
+          message => "#{routing}"
+          count => #{event_count}
+          type => "#{type}"
+        }
+      }
+      output {
+        elasticsearch {
+          host => "#{get_host()}"
+          port => "#{get_port('transport')}"
+          protocol => "transport"
+          index => "#{index}"
+          flush_size => #{flush_size}
+          routing => "%{message}"
+        }
+      }
+      CONFIG
+    }
+  end
+end

data/spec/integration/outputs/secure_spec.rb

@@ -0,0 +1,113 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "protocol" => "http",
+      "node_name" => "logstash",
+      "cluster" => "elasticsearch",
+      "host" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+      "ssl" => true,
+      "cacert" => "/tmp/ca/certs/cacert.pem",
+      # or
+      #"truststore" => "/tmp/ca/truststore.jks",
+      #"truststore_password" => "testeteste"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "protocol" => "http",
+      "cluster" => "elasticsearch",
+      "host" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "send messages to ElasticSearch using HTTPS", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "protocol" => "http",
+      "node_name" => "logstash",
+      "cluster" => "elasticsearch",
+      "host" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+      "ssl" => true,
+      "cacert" => "/tmp/ca/certs/cacert.pem",
+      # or
+      #"truststore" => "/tmp/ca/truststore.jks",
+      #"truststore_password" => "testeteste"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end
+
+describe "connect using HTTP Authentication", :elasticsearch_secure => true do
+  subject do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "protocol" => "http",
+      "cluster" => "elasticsearch",
+      "host" => "node01",
+      "user" => "user",
+      "password" => "changeme",
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    subject.register
+  end
+
+  it "sends events to ES" do
+    expect {
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+    }.to_not raise_error
+  end
+end

data/spec/integration/outputs/templates_spec.rb

@@ -0,0 +1,97 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "index template expected behavior", :integration => true do
+  ["transport", "http"].each do |protocol|
+    context "with protocol => #{protocol}" do
+        
+      subject! do
+        require "logstash/outputs/elasticsearch"
+        settings = {
+          "manage_template" => true,
+          "template_overwrite" => true,
+          "protocol" => protocol,
+          "host" => "#{get_host()}",
+          "port" => "#{get_port(protocol)}"
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      before :each do
+        # Delete all templates first.
+        require "elasticsearch"
+
+        # Clean ES of data before we start.
+        @es = get_client
+        @es.indices.delete_template(:name => "*")
+
+        # This can fail if there are no indexes, ignore failure.
+        @es.indices.delete(:index => "*") rescue nil
+
+        subject.register
+
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.receive(LogStash::Event.new("somevalue" => 100))
+        subject.receive(LogStash::Event.new("somevalue" => 10))
+        subject.receive(LogStash::Event.new("somevalue" => 1))
+        subject.receive(LogStash::Event.new("country" => "us"))
+        subject.receive(LogStash::Event.new("country" => "at"))
+        subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
+        subject.buffer_flush(:final => true)
+        @es.indices.refresh
+
+        # Wait or fail until everything's indexed.
+        Stud::try(20.times) do
+          r = @es.search
+          insist { r["hits"]["total"] } == 7
+        end
+      end
+
+      it "permits phrase searching on string fields" do
+        results = @es.search(:q => "message:\"sample message\"")
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+      end
+
+      it "numbers dynamically map to a numeric type and permit range queries" do
+        results = @es.search(:q => "somevalue:[5 TO 105]")
+        insist { results["hits"]["total"] } == 2
+
+        values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+        insist { values }.include?(10)
+        insist { values }.include?(100)
+        reject { values }.include?(1)
+      end
+
+      it "does not create .raw field for the message field" do
+        results = @es.search(:q => "message.raw:\"sample message here\"")
+        insist { results["hits"]["total"] } == 0
+      end
+
+      it "creates .raw field from any string field which is not_analyzed" do
+        results = @es.search(:q => "country.raw:\"us\"")
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+
+        # partial or terms should not work.
+        results = @es.search(:q => "country.raw:\"u\"")
+        insist { results["hits"]["total"] } == 0
+      end
+
+      it "make [geoip][location] a geo_point" do
+        results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
+      end
+
+      it "should index stopwords like 'at' " do
+        results = @es.search(:body => { "facets" => { "t" => { "terms" => { "field" => "country" } } } })["facets"]["t"]
+        terms = results["terms"].collect { |t| t["term"] }
+
+        insist { terms }.include?("us")
+
+        # 'at' is a stopword, make sure stopwords are not ignored.
+        insist { terms }.include?("at")
+      end
+    end
+  end
+end

data/spec/integration/outputs/transport_create_spec.rb

@@ -0,0 +1,94 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "transport client create actions", :integration => true do
+  require "logstash/outputs/elasticsearch"
+  require "elasticsearch"
+
+  def get_es_output(action, id = nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-create",
+      "template_overwrite" => true,
+      "protocol" => "transport",
+      "host" => get_host(),
+      "port" => get_port('transport'),
+      "action" => action
+    }
+    settings['document_id'] = id unless id.nil?
+    LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  context "when action => create" do
+    it "should create new documents with or without id" do
+      subject = get_es_output("create", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+
+    it "should create new documents without id" do
+      subject = get_es_output("create")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+  end
+
+  context "when action => create_unless_exists" do
+    it "should create new documents when specific id is specified" do
+      subject = get_es_output("create_unless_exists", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+
+    it "should fail to create a document when no id is specified" do
+      event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
+      action = ["create_unless_exists", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
+      subject = get_es_output(action[0])
+      subject.register
+      expect { subject.flush([action]) }.to raise_error
+    end
+
+    it "should unsuccesfully submit two records with the same document id" do
+      subject = get_es_output("create_unless_exists", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.receive(LogStash::Event.new("message" => "sample message here")) # 400 status failure (same id)
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+  end
+end

data/spec/{outputs → unit/outputs}/elasticsearch/protocol_spec.rb

@@ -49,4 +49,3 @@ describe LogStash::Outputs::Elasticsearch::Protocols::HTTPClient do
     end
   end
 end
-

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -0,0 +1,157 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "outputs/elasticsearch" do
+  context "registration" do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "elasticsearch").new("embedded" => "false", "protocol" => "transport", "manage_template" => "false")
+      # register will try to load jars and raise if it cannot find jars
+      expect {output.register}.to_not raise_error
+    end
+
+    it "should fail to register when protocol => http, action => create_unless_exists" do
+      output = LogStash::Plugin.lookup("output", "elasticsearch").new("protocol" => "http", "action" => "create_unless_exists")
+      expect {output.register}.to raise_error
+    end
+  end
+
+  describe "Authentication option" do
+    ["node", "transport"].each do |protocol|
+      context "with protocol => #{protocol}" do
+        subject do
+          require "logstash/outputs/elasticsearch"
+          settings = {
+            "protocol" => protocol,
+            "node_name" => "logstash",
+            "cluster" => "elasticsearch",
+            "host" => "node01",
+            "user" => "test",
+            "password" => "test"
+          }
+          next LogStash::Outputs::ElasticSearch.new(settings)
+        end
+
+        it "should fail in register" do
+          expect {subject.register}.to raise_error
+        end
+      end
+    end
+  end
+
+  describe "http client create" do
+    require "logstash/outputs/elasticsearch"
+    require "elasticsearch"
+
+    let(:options) {
+      {
+        "protocol" => "http",
+        "index" => "my-index",
+        "host" => "localhost",
+        "path" => "some-path"
+      }
+    }
+
+    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
+
+    let(:manticore_host) {
+      eso.client.first.send(:client).transport.options[:host].first
+    }
+
+    around(:each) do |block|
+      thread = eso.register
+      block.call()
+      thread.kill()
+    end
+
+    describe "with path" do
+      it "should properly create a URI with the path" do
+        expect(eso.path).to eql(options["path"])
+      end
+
+
+      it "should properly set the path on the HTTP client adding slashes" do
+        expect(manticore_host).to include("/" + options["path"] + "/")
+      end
+
+      context "with extra slashes" do
+        let(:path) { "/slashed-path/ "}
+        let(:eso) {
+          LogStash::Outputs::ElasticSearch.new(options.merge("path" => "/some-path/"))
+        }
+
+        it "should properly set the path on the HTTP client without adding slashes" do
+          expect(manticore_host).to include(options["path"])
+        end
+      end
+
+
+    end
+  end
+
+  describe "transport protocol" do
+    context "host not configured" do
+      subject do
+        require "logstash/outputs/elasticsearch"
+        settings = {
+          "protocol" => "transport",
+          "node_name" => "mynode"
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      it "should set host to localhost" do
+        expect(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:new).with({
+          :host => "localhost",
+          :port => "9300-9305",
+          :protocol => "transport",
+          :client_settings => {
+            "client.transport.sniff" => false,
+            "node.name" => "mynode"
+          }
+        })
+        subject.register
+      end
+    end
+
+    context "sniffing => true" do
+      subject do
+        require "logstash/outputs/elasticsearch"
+        settings = {
+          "host" => "node01",
+          "protocol" => "transport",
+          "sniffing" => true
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      it "should set the sniffing property to true" do
+        expect_any_instance_of(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:client).and_return(nil)
+        subject.register
+        client = subject.instance_eval("@current_client")
+        settings = client.instance_eval("@settings")
+
+        expect(settings.build.getAsMap["client.transport.sniff"]).to eq("true")
+      end
+    end
+
+    context "sniffing => false" do
+      subject do
+        require "logstash/outputs/elasticsearch"
+        settings = {
+          "host" => "node01",
+          "protocol" => "transport",
+          "sniffing" => false
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      it "should set the sniffing property to true" do
+        expect_any_instance_of(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:client).and_return(nil)
+        subject.register
+        client = subject.instance_eval("@current_client")
+        settings = client.instance_eval("@settings")
+
+        expect(settings.build.getAsMap["client.transport.sniff"]).to eq("false")
+      end
+    end
+  end
+end