logstash-output-elasticsearch-leprechaun-fork 1.0.8

data/spec/integration/outputs/templates_spec.rb

@@ -0,0 +1,97 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "index template expected behavior", :integration => true do
+  ["transport", "http"].each do |protocol|
+    context "with protocol => #{protocol}" do
+        
+      subject! do
+        require "logstash/outputs/elasticsearch"
+        settings = {
+          "manage_template" => true,
+          "template_overwrite" => true,
+          "protocol" => protocol,
+          "host" => "#{get_host()}",
+          "port" => "#{get_port(protocol)}"
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      before :each do
+        # Delete all templates first.
+        require "elasticsearch"
+
+        # Clean ES of data before we start.
+        @es = get_client
+        @es.indices.delete_template(:name => "*")
+
+        # This can fail if there are no indexes, ignore failure.
+        @es.indices.delete(:index => "*") rescue nil
+
+        subject.register
+
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.receive(LogStash::Event.new("somevalue" => 100))
+        subject.receive(LogStash::Event.new("somevalue" => 10))
+        subject.receive(LogStash::Event.new("somevalue" => 1))
+        subject.receive(LogStash::Event.new("country" => "us"))
+        subject.receive(LogStash::Event.new("country" => "at"))
+        subject.receive(LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] }))
+        subject.buffer_flush(:final => true)
+        @es.indices.refresh
+
+        # Wait or fail until everything's indexed.
+        Stud::try(20.times) do
+          r = @es.search
+          insist { r["hits"]["total"] } == 7
+        end
+      end
+
+      it "permits phrase searching on string fields" do
+        results = @es.search(:q => "message:\"sample message\"")
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+      end
+
+      it "numbers dynamically map to a numeric type and permit range queries" do
+        results = @es.search(:q => "somevalue:[5 TO 105]")
+        insist { results["hits"]["total"] } == 2
+
+        values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+        insist { values }.include?(10)
+        insist { values }.include?(100)
+        reject { values }.include?(1)
+      end
+
+      it "does not create .raw field for the message field" do
+        results = @es.search(:q => "message.raw:\"sample message here\"")
+        insist { results["hits"]["total"] } == 0
+      end
+
+      it "creates .raw field from any string field which is not_analyzed" do
+        results = @es.search(:q => "country.raw:\"us\"")
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+
+        # partial or terms should not work.
+        results = @es.search(:q => "country.raw:\"u\"")
+        insist { results["hits"]["total"] } == 0
+      end
+
+      it "make [geoip][location] a geo_point" do
+        results = @es.search(:body => { "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } })
+        insist { results["hits"]["total"] } == 1
+        insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
+      end
+
+      it "should index stopwords like 'at' " do
+        results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country" } } } })["aggregations"]["my_agg"]
+        terms = results["buckets"].collect { |b| b["key"] }
+
+        insist { terms }.include?("us")
+
+        # 'at' is a stopword, make sure stopwords are not ignored.
+        insist { terms }.include?("at")
+      end
+    end
+  end
+end

data/spec/integration/outputs/transport_create_spec.rb

@@ -0,0 +1,94 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "transport client create actions", :integration => true do
+  require "logstash/outputs/elasticsearch"
+  require "elasticsearch"
+
+  def get_es_output(action, id = nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-create",
+      "template_overwrite" => true,
+      "protocol" => "transport",
+      "host" => get_host(),
+      "port" => get_port('transport'),
+      "action" => action
+    }
+    settings['document_id'] = id unless id.nil?
+    LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  context "when action => create" do
+    it "should create new documents with or without id" do
+      subject = get_es_output("create", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+
+    it "should create new documents without id" do
+      subject = get_es_output("create")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+  end
+
+  context "when action => create_unless_exists" do
+    it "should create new documents when specific id is specified" do
+      subject = get_es_output("create_unless_exists", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+
+    it "should fail to create a document when no id is specified" do
+      event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
+      action = ["create_unless_exists", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
+      subject = get_es_output(action[0])
+      subject.register
+      expect { subject.flush([action]) }.to raise_error
+    end
+
+    it "should unsuccesfully submit two records with the same document id" do
+      subject = get_es_output("create_unless_exists", "id123")
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.receive(LogStash::Event.new("message" => "sample message here")) # 400 status failure (same id)
+      subject.buffer_flush(:final => true)
+      @es.indices.refresh
+      # Wait or fail until everything's indexed.
+      Stud::try(3.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
+    end
+  end
+end

data/spec/integration/outputs/update_spec.rb

@@ -0,0 +1,87 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "all protocols update actions", :integration => true do
+  require "logstash/outputs/elasticsearch"
+  require "elasticsearch"
+
+  def get_es_output( protocol, id = nil, upsert = nil, doc_as_upsert=nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "protocol" => protocol,
+      "host" => get_host(),
+      "port" => get_port(protocol),
+      "action" => "update"
+    }
+    settings['upsert'] = upsert unless upsert.nil?
+    settings['document_id'] = id unless id.nil?
+    settings['doc_as_upsert'] = doc_as_upsert unless doc_as_upsert.nil?
+    LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => 'logs',
+      :id => "123",
+      :body => { :message => 'Test' }
+    )
+    @es.indices.refresh
+  end
+
+  ["node", "transport", "http"].each do |protocol|
+    context "update only with #{protocol} protocol" do
+      it "should failed without a document_id" do
+        event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
+        action = ["update", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
+        subject = get_es_output(protocol)
+        subject.register
+        expect { subject.flush([action]) }.to raise_error
+      end
+
+      it "should not create new document" do
+        subject = get_es_output(protocol, "456")
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+
+      it "should update existing document" do
+        subject = get_es_output(protocol, "123")
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "updated message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["message"] } == 'updated message here'
+      end
+    end
+
+    context "upsert with #{protocol} protocol" do
+      it "should create new documents with upsert content" do
+        subject = get_es_output(protocol, "456", '{"message": "upsert message"}')
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'upsert message'
+      end
+
+      it "should create new documents with event/doc as upsert" do
+        subject = get_es_output(protocol, "456", nil, true)
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'sample message here'
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/protocol_spec.rb

@@ -0,0 +1,54 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/protocol"
+require "java"
+
+describe LogStash::Outputs::Elasticsearch::Protocols::NodeClient do
+  context "successful" do
+    it "should map correctly" do
+      index_response = org.elasticsearch.action.index.IndexResponse.new("my_index", "my_type", "my_id", 123, true)
+      update_response = org.elasticsearch.action.update.UpdateResponse.new("my_index", "my_type", "my_id", 123, false)
+      delete_response = org.elasticsearch.action.delete.DeleteResponse.new("my_index", "my_type", "my_id", 123, true)
+      bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", index_response)
+      bulk_item_response_update = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "update", update_response)
+      bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", delete_response)
+      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_update, bulk_item_response_delete], 0)
+      ret = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.normalize_bulk_response(bulk_response)
+      insist { ret } == {"errors" => false}
+    end
+  end
+
+  context "contains failures" do
+    it "should map correctly" do
+      failure = org.elasticsearch.action.bulk.BulkItemResponse::Failure.new("my_index", "my_type", "my_id", "error message", org.elasticsearch.rest.RestStatus::BAD_REQUEST)
+      bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", failure)
+      bulk_item_response_update = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "update", failure)
+      bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", failure)
+      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_update, bulk_item_response_delete], 0)
+      actual = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors" => true, "statuses" => [400, 400, 400], "error_messages" => ["error message", "error message", "error message"]}
+    end
+  end
+end
+
+describe LogStash::Outputs::Elasticsearch::Protocols::HTTPClient do
+  context "successful" do
+    it "should map correctly" do
+      bulk_response = {"took"=>74, "errors"=>false, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+                                                                          "_type"=>"logs", "_id"=>"AUxTS2C55Jrgi-hC6rQF",
+                                                                          "_version"=>1, "status"=>201}}]} 
+      actual = LogStash::Outputs::Elasticsearch::Protocols::HTTPClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors"=> false}
+    end
+  end
+
+  context "contains failures" do
+    it "should map correctly" do
+      bulk_response = {"took"=>71, "errors"=>true,
+                       "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+                                             "_type"=>"logs", "_id"=>"AUxTQ_OI5Jrgi-hC6rQB", "status"=>400,
+                                             "error"=>"MapperParsingException[failed to parse]..."}}]}
+      actual = LogStash::Outputs::Elasticsearch::Protocols::HTTPClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors"=> true, "statuses"=> [400], "error_messages" => ["MapperParsingException[failed to parse]..."]}
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -0,0 +1,59 @@
+require_relative "../../../spec/es_spec_helper"
+require 'stud/temporary'
+require 'elasticsearch'
+require "logstash/outputs/elasticsearch"
+
+describe "Proxy option" do
+  let(:settings) {
+    {
+      "protocol" => "http",
+      "host" => "node01",
+      "proxy" => proxy
+    }
+  }
+  subject {
+    LogStash::Outputs::ElasticSearch.new(settings)
+  }
+
+  before do
+    allow(::Elasticsearch::Client).to receive(:new).with(any_args)
+  end
+
+  describe "valid configs" do
+    before do
+      subject.register
+    end
+
+    context "when specified as a string" do
+      let(:proxy) { "http://127.0.0.1:1234" }
+
+      it "should set the proxy to the exact value" do
+        expect(::Elasticsearch::Client).to have_received(:new) do |options|
+          expect(options[:transport_options][:proxy]).to eql(proxy)
+        end
+      end
+    end
+
+    context "when specified as a hash" do
+      let(:proxy) { {"host" => "127.0.0.1", "protocol" => "http"} }
+
+      it "should pass through the proxy values as symbols" do
+        expected = {:host => proxy["host"], :protocol => proxy["protocol"]}
+        expect(::Elasticsearch::Client).to have_received(:new) do |options|
+          expect(options[:transport_options][:proxy]).to eql(expected)
+        end
+      end
+    end
+  end
+
+  describe "invalid configs" do
+    let(:proxy) { ["bad", "stuff"] }
+
+    it "should have raised an exception" do
+      expect {
+        subject.register
+      }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -0,0 +1,183 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "outputs/elasticsearch" do
+  context "registration" do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "elasticsearch").new("embedded" => "false", "protocol" => "transport", "manage_template" => "false")
+      # register will try to load jars and raise if it cannot find jars
+      expect {output.register}.to_not raise_error
+    end
+
+    it "should fail to register when protocol => http, action => create_unless_exists" do
+      output = LogStash::Plugin.lookup("output", "elasticsearch").new("protocol" => "http", "action" => "create_unless_exists")
+      expect {output.register}.to raise_error
+    end
+  end
+
+  describe "Authentication option" do
+    ["node", "transport"].each do |protocol|
+      context "with protocol => #{protocol}" do
+        subject do
+          require "logstash/outputs/elasticsearch"
+          settings = {
+            "protocol" => protocol,
+            "node_name" => "logstash",
+            "cluster" => "elasticsearch",
+            "host" => "node01",
+            "user" => "test",
+            "password" => "test"
+          }
+          next LogStash::Outputs::ElasticSearch.new(settings)
+        end
+
+        it "should fail in register" do
+          expect {subject.register}.to raise_error
+        end
+      end
+    end
+  end
+
+  describe "http client create" do
+    require "logstash/outputs/elasticsearch"
+    require "elasticsearch"
+
+    let(:options) {
+      {
+        "protocol" => "http",
+        "index" => "my-index",
+        "host" => "localhost",
+        "path" => "some-path"
+      }
+    }
+
+    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
+
+    let(:manticore_host) {
+      eso.client.first.send(:client).transport.options[:host].first
+    }
+
+    around(:each) do |block|
+      thread = eso.register
+      block.call()
+      thread.kill()
+    end
+
+    describe "with path" do
+      it "should properly create a URI with the path" do
+        expect(eso.path).to eql(options["path"])
+      end
+
+
+      it "should properly set the path on the HTTP client adding slashes" do
+        expect(manticore_host).to include("/" + options["path"] + "/")
+      end
+
+      context "with extra slashes" do
+        let(:path) { "/slashed-path/ "}
+        let(:eso) {
+          LogStash::Outputs::ElasticSearch.new(options.merge("path" => "/some-path/"))
+        }
+
+        it "should properly set the path on the HTTP client without adding slashes" do
+          expect(manticore_host).to include(options["path"])
+        end
+      end
+
+
+    end
+  end
+
+  describe "transport protocol" do
+    context "host not configured" do
+      subject do
+        require "logstash/outputs/elasticsearch"
+        settings = {
+          "protocol" => "transport",
+          "node_name" => "mynode"
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      it "should set host to localhost" do
+        expect(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:new).with({
+          :host => "localhost",
+          :port => "9300-9305",
+          :protocol => "transport",
+          :client_settings => {
+            "client.transport.sniff" => false,
+            "node.name" => "mynode"
+          }
+        })
+        subject.register
+      end
+    end
+
+    context "sniffing => true" do
+      subject do
+        require "logstash/outputs/elasticsearch"
+        settings = {
+          "host" => "node01",
+          "protocol" => "transport",
+          "sniffing" => true
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      it "should set the sniffing property to true" do
+        expect_any_instance_of(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:client).and_return(nil)
+        subject.register
+        client = subject.instance_eval("@current_client")
+        settings = client.instance_eval("@settings")
+
+        expect(settings.build.getAsMap["client.transport.sniff"]).to eq("true")
+      end
+    end
+  end
+
+  context "allow_mapping_mangling" do
+    context 'elasticsearch borks on mapping mismatch' do
+      context 'logstash was configured to allow mapping mangling' do
+        subject do
+          require "logstash/outputs/elasticsearch"
+          settings = {
+            "host" => "node01",
+            "protocol" => "transport"
+          }
+          next LogStash::Outputs::ElasticSearch.new(settings)
+        end
+
+        it 'should identify mapping mismatch errors correctly' do
+          error_message = "MapperParsingException[failed to parse]...dsdfsf"
+          expect_any_instance_of(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:client).and_return(nil)
+          subject.register
+          client = subject.instance_eval("@current_client")
+        end
+
+        it 'should retry with a semi-randomized type when configured to do so' do
+          true
+        end
+      end
+    end
+  end
+
+    context "sniffing => false" do
+      subject do
+        require "logstash/outputs/elasticsearch"
+        settings = {
+          "host" => "node01",
+          "protocol" => "transport",
+          "sniffing" => false
+        }
+        next LogStash::Outputs::ElasticSearch.new(settings)
+      end
+
+      it "should set the sniffing property to true" do
+        expect_any_instance_of(LogStash::Outputs::Elasticsearch::Protocols::TransportClient).to receive(:client).and_return(nil)
+        subject.register
+        client = subject.instance_eval("@current_client")
+        settings = client.instance_eval("@settings")
+
+        expect(settings.build.getAsMap["client.transport.sniff"]).to eq("false")
+      end
+    end
+  end