logstash-output-elasticsearch-test 10.3.0-x86_64-linux

data/spec/integration/outputs/parent_spec.rb

@@ -0,0 +1,102 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+
+if ESHelper.es_version_satisfies?(">= 5.6")
+  context "when using elasticsearch 5.6 and above", :integration => true do
+
+    shared_examples "a join field based parent indexer" do
+      let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+
+      let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+
+      let(:event_count) { 10000 + rand(500) }
+      let(:parent) { "not_implemented" }
+      let(:config) { "not_implemented" }
+      let(:parent_id) { "test" }
+      let(:join_field) { "join_field" }
+      let(:parent_relation) { "parent_type" }
+      let(:child_relation) { "child_type" }
+      let(:default_headers) {
+        {"Content-Type" => "application/json"}
+      }
+      subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+      before do
+        # Add mapping and a parent document
+        index_url = "http://#{get_host_port()}/#{index}"
+
+        properties = {
+          "properties" => {
+            join_field => {
+              "type" => "join",
+              "relations" => { parent_relation => child_relation }
+            }
+          }
+        }
+
+        mapping = ESHelper.es_version_satisfies?('<7') ? { "mappings" => { type => properties } }
+                                                       : { "mappings" => properties}
+
+        if ESHelper.es_version_satisfies?('<6')
+          mapping.merge!({
+                 "settings" => {
+                   "mapping.single_type" => true
+                 }})
+        end
+        Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
+        pdoc = { "message" => "ohayo", join_field => parent_relation }
+        Manticore.put("#{index_url}/#{type}/#{parent_id}", {:body => pdoc.to_json, :headers => default_headers}).call
+
+        subject.register
+        subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => parent_id, "message" => "Hello World!", join_field => child_relation) })
+      end
+
+
+      it "ships events" do
+        index_url = "http://#{get_host_port()}/#{index}"
+
+        Manticore.post("#{index_url}/_refresh").call
+
+        # Wait until all events are available.
+        Stud::try(10.times) do
+          query = { "query" => { "has_parent" => { "parent_type" => parent_relation, "query" => { "match_all" => { } } } } }
+          response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
+          data = response.body
+          result = LogStash::Json.load(data)
+          cur_count = result["count"]
+          expect(cur_count).to eq(event_count)
+        end
+      end
+    end
+
+    describe "(http protocol) index events with static parent" do
+      it_behaves_like 'a join field based parent indexer' do
+        let(:config) {
+          {
+            "hosts" => get_host_port,
+            "index" => index,
+            "parent" => parent_id,
+            "document_type" => type,
+            "join_field" => join_field,
+            "manage_template" => false
+          }
+        }
+      end
+    end
+
+    describe "(http_protocol) index events with fieldref in parent value" do
+      it_behaves_like 'a join field based parent indexer' do
+        let(:config) {
+          {
+            "hosts" => get_host_port,
+            "index" => index,
+            "parent" => "%{link_to}",
+            "document_type" => type,
+            "join_field" => join_field,
+            "manage_template" => false
+          }
+        }
+      end
+    end
+  end
+end

data/spec/integration/outputs/retry_spec.rb

@@ -0,0 +1,169 @@
+require "logstash/outputs/elasticsearch"
+require_relative "../../../spec/es_spec_helper"
+
+describe "failures in bulk class expected behavior", :integration => true do
+  let(:template) { '{"template" : "not important, will be updated by :index"}' }
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:action1) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1]) }
+  let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
+  let(:action2) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2]) }
+  let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+
+  def mock_actions_with_response(*resp)
+    raise ArgumentError, "Cannot mock actions until subject is registered and has a client!" unless subject.client
+
+    expanded_responses = resp.map do |resp|
+      items = resp["statuses"] && resp["statuses"].map do |status|
+        {"create" => {"status" => status, "error" => "Error for #{status}"}}
+      end
+
+      {
+        "errors" => resp["errors"],
+        "items" => items
+      }
+    end
+
+    allow(subject.client).to receive(:bulk).and_return(*expanded_responses)
+  end
+
+  subject! do
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-2014.11.17",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "retry_max_interval" => 64,
+      "retry_initial_interval" => 2
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
+    allow(Stud).to receive(:stoppable_sleep)
+
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
+    @es.indices.delete(:index => "*")
+    @es.indices.refresh
+  end
+
+  after :each do
+    subject.close
+  end
+
+  it "should retry exactly once if all bulk actions are successful" do
+    expect(subject).to receive(:submit).with([action1, action2]).once.and_call_original
+    subject.register
+    mock_actions_with_response({"errors" => false})
+    subject.multi_receive([event1, event2])
+  end
+
+  it "retry exceptions within the submit body" do
+    call_count = 0
+    subject.register
+
+    expect(subject.client).to receive(:bulk).with(anything).exactly(3).times do
+      if (call_count += 1) <= 2
+        raise "error first two times"
+      else
+        {"errors" => false}
+      end
+    end
+
+    subject.multi_receive([event1])
+  end
+
+  it "should retry actions with response status of 503" do    expect(subject).to receive(:submit).with([action1, action1, action1, action2]).ordered.once.and_call_original
+    expect(subject).to receive(:submit).with([action1, action2]).ordered.once.and_call_original
+    expect(subject).to receive(:submit).with([action2]).ordered.once.and_call_original
+
+    subject.register
+    mock_actions_with_response({"errors" => true, "statuses" => [200, 200, 503, 503]},
+                               {"errors" => true, "statuses" => [200, 503]},
+                               {"errors" => false})
+
+    subject.multi_receive([event1, event1, event1, event2])
+  end
+
+  retryable_codes = [429, 502, 503]
+
+  retryable_codes.each do |code|
+    it "should retry actions with response status of #{code}" do
+      subject.register
+
+      mock_actions_with_response({"errors" => true, "statuses" => [code]},
+                                 {"errors" => false})
+      expect(subject).to receive(:submit).with([action1]).twice.and_call_original
+
+      subject.multi_receive([event1])
+    end
+  end
+
+  it "should retry an event infinitely until a non retryable status occurs" do
+    expect(subject).to receive(:submit).with([action1]).exactly(6).times.and_call_original
+    subject.register
+
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [400]})
+
+    subject.multi_receive([event1])
+  end
+
+  it "should sleep for an exponentially increasing amount of time on each retry, capped by the max" do
+    [2, 4, 8, 16, 32, 64, 64].each_with_index do |interval,i|
+      expect(Stud).to receive(:stoppable_sleep).with(interval).ordered
+    end
+
+    subject.register
+
+    mock_actions_with_response({"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [429]},
+                               {"errors" => true, "statuses" => [400]})
+
+    subject.multi_receive([event1])
+  end
+
+  it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunately)" do
+    subject.register
+    expect(subject).to receive(:submit).once.and_call_original
+    subject.multi_receive([invalid_event])
+    subject.close
+
+    @es.indices.refresh
+    r = @es.search
+    expect(r).to have_hits(0)
+  end
+
+  it "successful requests should not be appended to retry queue" do
+    expect(subject).to receive(:submit).once.and_call_original
+
+    subject.register
+    subject.multi_receive([event1])
+    subject.close
+    @es.indices.refresh
+    r = @es.search
+    expect(r).to have_hits(1)
+  end
+
+  it "should only index proper events" do
+    subject.register
+    subject.multi_receive([invalid_event, event1])
+    subject.close
+
+    @es.indices.refresh
+    r = @es.search
+    expect(r).to have_hits(1)
+  end
+end

data/spec/integration/outputs/routing_spec.rb

@@ -0,0 +1,61 @@
+require_relative "../../../spec/es_spec_helper"
+
+shared_examples "a routing indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:routing) { "not_implemented" }
+    let(:config) { "not_implemented" }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+    before do
+      subject.register
+      event_count.times do
+        subject.multi_receive([LogStash::Event.new("message" => "test", "type" => type)])
+      end
+    end
+
+
+    it "ships events" do
+      index_url = "http://#{get_host_port()}/#{index}"
+
+      client = Manticore::Client.new
+      client.post("#{index_url}/_refresh").call
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        data = ""
+
+        response = client.get("#{index_url}/_count?q=*&routing=#{routing}").call
+        result = LogStash::Json.load(response.body)
+        cur_count = result["count"]
+        expect(cur_count).to eq(event_count)
+      end
+    end
+end
+
+describe "(http protocol) index events with static routing", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "routing" => routing
+      }
+    }
+  end
+end
+
+describe "(http_protocol) index events with fieldref in routing value", :integration => true do
+  it_behaves_like 'a routing indexer' do
+    let(:routing) { "test" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "routing" => "%{message}"
+      }
+    }
+  end
+end

data/spec/integration/outputs/sniffer_spec.rb

@@ -0,0 +1,133 @@
+require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "json"
+require "socket"
+
+describe "pool sniffer", :integration => true do
+  let(:logger) { Cabin::Channel.get }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:es_host) { get_host_port.split(":").first }
+  let(:es_port) { get_host_port.split(":").last }
+  let(:es_ip) { IPSocket.getaddress(es_host) }
+  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
+  let(:options) do
+    {
+      :resurrect_delay => 2, # Shorten the delay a bit to speed up tests
+      :url_normalizer => proc {|u| u},
+      :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy)
+    }
+  end
+
+  subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
+
+  describe("Simple sniff parsing")  do
+    before(:each) { subject.start }
+
+    context "with single node" do
+      it "should execute a sniff without error" do
+        expect { subject.check_sniff }.not_to raise_error
+      end
+
+      it "should return single sniff URL" do
+        uris = subject.check_sniff
+
+        expect(uris.size).to eq(1)
+      end
+
+      it "should return the correct sniff URL" do
+        if ESHelper.es_version_satisfies?(">= 2", "<7")
+          # We do a more thorough check on these versions because we can more reliably guess the ip
+          uris = subject.check_sniff
+
+          expect(uris).to include(::LogStash::Util::SafeURI.new("//#{es_ip}:#{es_port}"))
+        else
+          # ES 1.x (and ES 7.x) returned the public hostname by default. This is hard to approximate
+          # so for ES1.x and 7.x we don't check the *exact* hostname
+          skip
+        end
+      end
+    end
+  end
+
+  if ESHelper.es_version_satisfies?("<= 2")
+    describe("Complex sniff parsing ES 2x/1x") do
+      before(:each) do
+        response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/2x_1x.json"))
+        allow(subject).to receive(:perform_request).and_return([nil, { version: "2.0" }, response_double])
+        subject.start
+      end
+
+      context "with multiple nodes but single http-enabled data node" do
+        it "should execute a sniff without error" do
+          expect { subject.check_sniff }.not_to raise_error
+        end
+
+        it "should return one sniff URL" do
+          uris = subject.check_sniff
+
+          expect(uris.size).to eq(1)
+        end
+
+        it "should return the correct sniff URL" do
+          if ESHelper.es_version_satisfies?(">= 2")
+            # We do a more thorough check on these versions because we can more reliably guess the ip
+            uris = subject.check_sniff
+
+            expect(uris).to include(::LogStash::Util::SafeURI.new("http://localhost:9201"))
+          else
+            # ES 1.x returned the public hostname by default. This is hard to approximate
+            # so for ES1.x we don't check the *exact* hostname
+            skip
+          end
+        end
+      end
+    end
+  end
+
+
+  if ESHelper.es_version_satisfies?(">= 7")
+    describe("Complex sniff parsing ES 7x") do
+      before(:each) do
+        response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/7x.json"))
+        allow(subject).to receive(:perform_request).and_return([nil, { version: "7.0" }, response_double])
+        subject.start
+      end
+
+      context "with mixed master-only, data-only, and data + master nodes" do
+        it "should execute a sniff without error" do
+          expect { subject.check_sniff }.not_to raise_error
+        end
+
+        it "should return the correct sniff URLs" do
+          # ie. with the master-only node, and with the node name correctly set.
+          uris = subject.check_sniff
+
+          expect(uris).to include(::LogStash::Util::SafeURI.new("//dev-masterdata:9201"), ::LogStash::Util::SafeURI.new("//dev-data:9202"))
+        end
+      end
+    end
+  end
+  if ESHelper.es_version_satisfies?(">= 5")
+    describe("Complex sniff parsing ES 6x/5x") do
+      before(:each) do
+        response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/5x_6x.json"))
+        allow(subject).to receive(:perform_request).and_return([nil, { version: "5.0" }, response_double])
+        subject.start
+      end
+
+      context "with mixed master-only, data-only, and data + master nodes" do
+        it "should execute a sniff without error" do
+          expect { subject.check_sniff }.not_to raise_error
+        end
+
+        it "should return the correct sniff URLs" do
+          # ie. without the master-only node
+          uris = subject.check_sniff
+
+          expect(uris).to include(::LogStash::Util::SafeURI.new("//127.0.0.1:9201"), ::LogStash::Util::SafeURI.new("//127.0.0.1:9202"), ::LogStash::Util::SafeURI.new("//127.0.0.1:9203"))
+        end
+      end
+    end
+  end
+end

data/spec/integration/outputs/templates_5x_spec.rb

@@ -0,0 +1,98 @@
+require_relative "../../../spec/es_spec_helper"
+
+if ESHelper.es_version_satisfies?(">= 5")
+  describe "index template expected behavior for 5.x", :integration => true do
+    subject! do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "manage_template" => true,
+        "template_overwrite" => true,
+        "hosts" => "#{get_host_port()}"
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
+
+    before :each do
+      # Delete all templates first.
+      require "elasticsearch"
+
+      # Clean ES of data before we start.
+      @es = get_client
+      @es.indices.delete_template(:name => "*")
+
+      # This can fail if there are no indexes, ignore failure.
+      @es.indices.delete(:index => "*") rescue nil
+
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @es.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @es.search
+        expect(r).to have_hits(8)
+      end
+    end
+
+    it "permits phrase searching on string fields" do
+      results = @es.search(:q => "message:\"sample message\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
+    end
+
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @es.search(:q => "somevalue:[5 TO 105]")
+      expect(results).to have_hits(2)
+
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      expect(values).to include(10)
+      expect(values).to include(100)
+      expect(values).to_not include(1)
+    end
+
+    it "does not create .keyword field for top-level message field" do
+      results = @es.search(:q => "message.keyword:\"sample message here\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "creates .keyword field for nested message fields" do
+      results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+      expect(results).to have_hits(1)
+    end
+
+    it "creates .keyword field from any string field which is not_analyzed" do
+      results = @es.search(:q => "country.keyword:\"us\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+
+      # partial or terms should not work.
+      results = @es.search(:q => "country.keyword:\"u\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "make [geoip][location] a geo_point" do
+      expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+    end
+
+    it "aggregate .keyword results correctly " do
+      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
+
+      expect(terms).to include("us")
+
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      expect(terms).to include("at")
+    end
+  end
+end

data/spec/integration/outputs/templates_spec.rb

@@ -0,0 +1,98 @@
+require_relative "../../../spec/es_spec_helper"
+
+if ESHelper.es_version_satisfies?("< 5")
+  describe "index template expected behavior", :integration => true do
+    subject! do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "manage_template" => true,
+        "template_overwrite" => true,
+        "hosts" => "#{get_host_port()}"
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
+
+    before :each do
+      # Delete all templates first.
+      require "elasticsearch"
+
+      # Clean ES of data before we start.
+      @es = get_client
+      @es.indices.delete_template(:name => "*")
+
+      # This can fail if there are no indexes, ignore failure.
+      @es.indices.delete(:index => "*") rescue nil
+
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @es.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @es.search
+        expect(r).to have_hits(8)
+      end
+    end
+
+    it "permits phrase searching on string fields" do
+      results = @es.search(:q => "message:\"sample message\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
+    end
+
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @es.search(:q => "somevalue:[5 TO 105]")
+      expect(results).to have_hits(2)
+
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      expect(values).to include(10)
+      expect(values).to include(100)
+      expect(values).to_not include(1)
+    end
+
+    it "does not create .raw field for the message field" do
+      results = @es.search(:q => "message.raw:\"sample message here\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "creates .raw field for nested message fields" do
+      results = @es.search(:q => "somemessage.message.raw:\"sample nested message here\"")
+      expect(results).to have_hits(1)
+    end
+
+    it "creates .raw field from any string field which is not_analyzed" do
+      results = @es.search(:q => "country.raw:\"us\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+
+      # partial or terms should not work.
+      results = @es.search(:q => "country.raw:\"u\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "make [geoip][location] a geo_point" do
+      expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
+    end
+
+    it "aggregate .raw results correctly " do
+      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.raw" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
+
+      expect(terms).to include("us")
+
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      expect(terms).to include("at")
+    end
+  end
+end