logstash-output-elasticsearch 7.4.3-java → 8.0.0-java

data/spec/integration/outputs/parent_spec.rb

@@ -1,165 +1,71 @@
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
-if ESHelper.es_version_satisfies?("< 6")
-  context "when using elasticsearch 5.x and before", :integration => true do
-    shared_examples "a type based parent indexer" do
-      let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-      let(:type) { 10.times.collect { rand(10).to_s }.join("") }
-      let(:event_count) { 10000 + rand(500) }
-      let(:parent) { "not_implemented" }
-      let(:config) { "not_implemented" }
-      let(:default_headers) {
-        {"Content-Type" => "application/json"}
-      }
-      subject { LogStash::Outputs::ElasticSearch.new(config) }
-
-      before do
-        # Add mapping and a parent document
-        index_url = "http://#{get_host_port()}/#{index}"
-        mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
-        Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
-        pdoc = { "foo" => "bar" }
-        Manticore.put("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers}).call
-
-        subject.register
-        subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
-      end
-
+shared_examples "a parent indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:parent) { "not_implemented" }
+    let(:config) { "not_implemented" }
+    let(:default_headers) {
+      {"Content-Type" => "application/json"}
+    }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+    before do
+      # Add mapping and a parent document
+      index_url = "http://#{get_host_port()}/#{index}"
+      ftw = FTW::Agent.new
+      mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
+      ftw.put!("#{index_url}", {:body => mapping.to_json, :headers => default_headers})
+      pdoc = { "foo" => "bar" }
+      ftw.put!("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers})
+
+      subject.register
+      subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
+    end
 
-      it "ships events" do
-        index_url = "http://#{get_host_port()}/#{index}"
 
-        Manticore.post("#{index_url}/_refresh").call
+    it "ships events" do
+      index_url = "http://#{get_host_port()}/#{index}"
 
-        # Wait until all events are available.
-        Stud::try(10.times) do
-          query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
-          response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
-          data = response.body
-          result = LogStash::Json.load(data)
-          cur_count = result["count"]
-          insist { cur_count } == event_count
-        end
-      end
-    end
+      ftw = FTW::Agent.new
+      ftw.post!("#{index_url}/_refresh")
 
-    describe "(http protocol) index events with static parent" do
-      it_behaves_like 'a type based parent indexer' do
-        let(:parent) { "test" }
-        let(:config) {
-          {
-            "hosts" => get_host_port,
-            "index" => index,
-            "parent" => parent
-          }
-        }
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
+        data = ""
+        response = ftw.post!("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
+        response.read_body { |chunk| data << chunk }
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
       end
     end
+end
 
-    describe "(http_protocol) index events with fieldref in parent value" do
-      it_behaves_like 'a type based parent indexer' do
-        let(:config) {
-          {
-            "hosts" => get_host_port,
-            "index" => index,
-            "parent" => "%{link_to}"
-          }
-        }
-      end
-    end
+describe "(http protocol) index events with static parent", :integration => true do
+  it_behaves_like 'a parent indexer' do
+    let(:parent) { "test" }
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "parent" => parent
+      }
+    }
   end
 end
 
-if ESHelper.es_version_satisfies?(">= 5.6")
-  context "when using elasticsearch 5.6 and above", :integration => true do
-    shared_examples "a join field based parent indexer" do
-      let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-      let(:type) { 10.times.collect { rand(10).to_s }.join("") }
-      let(:event_count) { 10000 + rand(500) }
-      let(:parent) { "not_implemented" }
-      let(:config) { "not_implemented" }
-      let(:parent_id) { "test" }
-      let(:join_field) { "join_field" }
-      let(:parent_relation) { "parent_type" }
-      let(:child_relation) { "child_type" }
-      let(:default_headers) {
-        {"Content-Type" => "application/json"}
+describe "(http_protocol) index events with fieldref in parent value", :integration => true do
+  it_behaves_like 'a parent indexer' do
+    let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index,
+        "parent" => "%{link_to}"
       }
-      subject { LogStash::Outputs::ElasticSearch.new(config) }
-
-      before do
-        # Add mapping and a parent document
-        index_url = "http://#{get_host_port()}/#{index}"
-        mapping = {
-          "mappings" => {
-            type => {
-              "properties" => {
-                join_field => {
-                  "type" => "join",
-                  "relations" => { parent_relation => child_relation }
-                }
-              }
-            }
-          }
-        }
-        if ESHelper.es_version_satisfies?('<6')
-          mapping.merge!({ "settings" => { "mapping.single_type" => true }})
-        end
-        Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
-        pdoc = { "message" => "ohayo", join_field => parent_relation }
-        Manticore.put("#{index_url}/#{type}/#{parent_id}", {:body => pdoc.to_json, :headers => default_headers}).call
-
-        subject.register
-        subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => parent_id, "message" => "Hello World!", join_field => child_relation) })
-      end
-
-
-      it "ships events" do
-        index_url = "http://#{get_host_port()}/#{index}"
-
-        Manticore.post("#{index_url}/_refresh").call
-
-        # Wait until all events are available.
-        Stud::try(10.times) do
-          query = { "query" => { "has_parent" => { "parent_type" => parent_relation, "query" => { "match_all" => { } } } } }
-          response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
-          data = response.body
-          result = LogStash::Json.load(data)
-          cur_count = result["count"]
-          insist { cur_count } == event_count
-        end
-      end
-    end
-
-    describe "(http protocol) index events with static parent" do
-      it_behaves_like 'a join field based parent indexer' do
-        let(:config) {
-          {
-            "hosts" => get_host_port,
-            "index" => index,
-            "parent" => parent_id,
-            "document_type" => type,
-            "join_field" => join_field,
-            "manage_template" => false
-          }
-        }
-      end
-    end
-
-    describe "(http_protocol) index events with fieldref in parent value" do
-      it_behaves_like 'a join field based parent indexer' do
-        let(:config) {
-          {
-            "hosts" => get_host_port,
-            "index" => index,
-            "parent" => "%{link_to}",
-            "document_type" => type,
-            "join_field" => join_field,
-            "manage_template" => false
-          }
-        }
-      end
-    end
+    }
   end
 end

data/spec/integration/outputs/sniffer_spec.rb

@@ -38,10 +38,7 @@ describe "pool sniffer", :integration => true do
   end
   
   # We do a more thorough check on these versions because we can more reliably guess the ip
-
-  if ESHelper.es_version_satisfies?(">= 2")
-    describe("Complex sniff parsing ES 6x/5x/2x") do
-      include_examples("sniff parsing", true)
-    end
+  describe("Complex sniff parsing ES 6x/5x/2x", :version_greater_than_equal_to_2x => true) do
+    include_examples("sniff parsing", true)
   end
 end

data/spec/integration/outputs/templates_5x_spec.rb

@@ -1,98 +1,99 @@
 require_relative "../../../spec/es_spec_helper"
 
-if ESHelper.es_version_satisfies?(">= 5")
-  describe "index template expected behavior for 5.x", :integration => true do
-    subject! do
-      require "logstash/outputs/elasticsearch"
-      settings = {
-        "manage_template" => true,
-        "template_overwrite" => true,
-        "hosts" => "#{get_host_port()}"
-      }
-      next LogStash::Outputs::ElasticSearch.new(settings)
-    end
+# This file is a copy of template test for 2.x. We can DRY this up later.
+describe "index template expected behavior for 5.x", :integration => true, :version_greater_than_equal_to_5x => true do
+  subject! do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "manage_template" => true,
+      "template_overwrite" => true,
+      "hosts" => "#{get_host_port()}"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
 
-    before :each do
-      # Delete all templates first.
-      require "elasticsearch"
-
-      # Clean ES of data before we start.
-      @es = get_client
-      @es.indices.delete_template(:name => "*")
-
-      # This can fail if there are no indexes, ignore failure.
-      @es.indices.delete(:index => "*") rescue nil
-
-      subject.register
-
-      subject.multi_receive([
-        LogStash::Event.new("message" => "sample message here"),
-        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-        LogStash::Event.new("somevalue" => 100),
-        LogStash::Event.new("somevalue" => 10),
-        LogStash::Event.new("somevalue" => 1),
-        LogStash::Event.new("country" => "us"),
-        LogStash::Event.new("country" => "at"),
-        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-      ])
-
-      @es.indices.refresh
-
-      # Wait or fail until everything's indexed.
-      Stud::try(20.times) do
-        r = @es.search
-        insist { r["hits"]["total"] } == 8
-      end
-    end
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
 
-    it "permits phrase searching on string fields" do
-      results = @es.search(:q => "message:\"sample message\"")
-      insist { results["hits"]["total"] } == 1
-      insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
-    end
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
 
-    it "numbers dynamically map to a numeric type and permit range queries" do
-      results = @es.search(:q => "somevalue:[5 TO 105]")
-      insist { results["hits"]["total"] } == 2
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
 
-      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-      insist { values }.include?(10)
-      insist { values }.include?(100)
-      reject { values }.include?(1)
-    end
+    subject.register
 
-    it "does not create .keyword field for top-level message field" do
-      results = @es.search(:q => "message.keyword:\"sample message here\"")
-      insist { results["hits"]["total"] } == 0
-    end
+    subject.multi_receive([
+      LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+      LogStash::Event.new("somevalue" => 100),
+      LogStash::Event.new("somevalue" => 10),
+      LogStash::Event.new("somevalue" => 1),
+      LogStash::Event.new("country" => "us"),
+      LogStash::Event.new("country" => "at"),
+      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+    ])
 
-    it "creates .keyword field for nested message fields" do
-      results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
-      insist { results["hits"]["total"] } == 1
+    @es.indices.refresh
+
+    # Wait or fail until everything's indexed.
+    Stud::try(20.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 8
     end
+  end
 
-    it "creates .keyword field from any string field which is not_analyzed" do
-      results = @es.search(:q => "country.keyword:\"us\"")
-      insist { results["hits"]["total"] } == 1
-      insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+  it "permits phrase searching on string fields" do
+    results = @es.search(:q => "message:\"sample message\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+  end
 
-      # partial or terms should not work.
-      results = @es.search(:q => "country.keyword:\"u\"")
-      insist { results["hits"]["total"] } == 0
-    end
+  it "numbers dynamically map to a numeric type and permit range queries" do
+    results = @es.search(:q => "somevalue:[5 TO 105]")
+    insist { results["hits"]["total"] } == 2
 
-    it "make [geoip][location] a geo_point" do
-      expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
-    end
+    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+    insist { values }.include?(10)
+    insist { values }.include?(100)
+    reject { values }.include?(1)
+  end
 
-    it "aggregate .keyword results correctly " do
-      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
-      terms = results["buckets"].collect { |b| b["key"] }
+  it "does not create .keyword field for top-level message field" do
+    results = @es.search(:q => "message.keyword:\"sample message here\"")
+    insist { results["hits"]["total"] } == 0
+  end
 
-      insist { terms }.include?("us")
+  it "creates .keyword field for nested message fields" do
+    results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+    insist { results["hits"]["total"] } == 1
+  end
 
-      # 'at' is a stopword, make sure stopwords are not ignored.
-      insist { terms }.include?("at")
-    end
+  it "creates .keyword field from any string field which is not_analyzed" do
+    results = @es.search(:q => "country.keyword:\"us\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+
+    # partial or terms should not work.
+    results = @es.search(:q => "country.keyword:\"u\"")
+    insist { results["hits"]["total"] } == 0
+  end
+
+  it "make [geoip][location] a geo_point" do
+    expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
+  end
+
+  it "aggregate .keyword results correctly " do
+    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+    terms = results["buckets"].collect { |b| b["key"] }
+
+    insist { terms }.include?("us")
+
+    # 'at' is a stopword, make sure stopwords are not ignored.
+    insist { terms }.include?("at")
   end
 end
+
+

data/spec/integration/outputs/templates_spec.rb

@@ -1,98 +1,98 @@
 require_relative "../../../spec/es_spec_helper"
 
-if ESHelper.es_version_satisfies?("< 5")
-  describe "index template expected behavior", :integration => true do
-    subject! do
-      require "logstash/outputs/elasticsearch"
-      settings = {
-        "manage_template" => true,
-        "template_overwrite" => true,
-        "hosts" => "#{get_host_port()}"
-      }
-      next LogStash::Outputs::ElasticSearch.new(settings)
-    end
+describe "index template expected behavior", :integration => true, :version_less_than_5x => true do
+  subject! do
+    require "logstash/outputs/elasticsearch"
+    settings = {
+      "manage_template" => true,
+      "template_overwrite" => true,
+      "hosts" => "#{get_host_port()}"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
 
-    before :each do
-      # Delete all templates first.
-      require "elasticsearch"
-
-      # Clean ES of data before we start.
-      @es = get_client
-      @es.indices.delete_template(:name => "*")
-
-      # This can fail if there are no indexes, ignore failure.
-      @es.indices.delete(:index => "*") rescue nil
-
-      subject.register
-
-      subject.multi_receive([
-        LogStash::Event.new("message" => "sample message here"),
-        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-        LogStash::Event.new("somevalue" => 100),
-        LogStash::Event.new("somevalue" => 10),
-        LogStash::Event.new("somevalue" => 1),
-        LogStash::Event.new("country" => "us"),
-        LogStash::Event.new("country" => "at"),
-        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-      ])
-
-      @es.indices.refresh
-
-      # Wait or fail until everything's indexed.
-      Stud::try(20.times) do
-        r = @es.search
-        insist { r["hits"]["total"] } == 8
-      end
-    end
+  before :each do
+    # Delete all templates first.
+    require "elasticsearch"
 
-    it "permits phrase searching on string fields" do
-      results = @es.search(:q => "message:\"sample message\"")
-      insist { results["hits"]["total"] } == 1
-      insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
-    end
+    # Clean ES of data before we start.
+    @es = get_client
+    @es.indices.delete_template(:name => "*")
 
-    it "numbers dynamically map to a numeric type and permit range queries" do
-      results = @es.search(:q => "somevalue:[5 TO 105]")
-      insist { results["hits"]["total"] } == 2
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
 
-      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-      insist { values }.include?(10)
-      insist { values }.include?(100)
-      reject { values }.include?(1)
-    end
+    subject.register
 
-    it "does not create .raw field for the message field" do
-      results = @es.search(:q => "message.raw:\"sample message here\"")
-      insist { results["hits"]["total"] } == 0
-    end
+    subject.multi_receive([
+      LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+      LogStash::Event.new("somevalue" => 100),
+      LogStash::Event.new("somevalue" => 10),
+      LogStash::Event.new("somevalue" => 1),
+      LogStash::Event.new("country" => "us"),
+      LogStash::Event.new("country" => "at"),
+      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+    ])
 
-    it "creates .raw field for nested message fields" do
-      results = @es.search(:q => "somemessage.message.raw:\"sample nested message here\"")
-      insist { results["hits"]["total"] } == 1
+    @es.indices.refresh
+
+    # Wait or fail until everything's indexed.
+    Stud::try(20.times) do
+      r = @es.search
+      insist { r["hits"]["total"] } == 8
     end
+  end
 
-    it "creates .raw field from any string field which is not_analyzed" do
-      results = @es.search(:q => "country.raw:\"us\"")
-      insist { results["hits"]["total"] } == 1
-      insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+  it "permits phrase searching on string fields" do
+    results = @es.search(:q => "message:\"sample message\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+  end
 
-      # partial or terms should not work.
-      results = @es.search(:q => "country.raw:\"u\"")
-      insist { results["hits"]["total"] } == 0
-    end
+  it "numbers dynamically map to a numeric type and permit range queries" do
+    results = @es.search(:q => "somevalue:[5 TO 105]")
+    insist { results["hits"]["total"] } == 2
 
-    it "make [geoip][location] a geo_point" do
-      expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
-    end
+    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+    insist { values }.include?(10)
+    insist { values }.include?(100)
+    reject { values }.include?(1)
+  end
 
-    it "aggregate .raw results correctly " do
-      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.raw" } } } })["aggregations"]["my_agg"]
-      terms = results["buckets"].collect { |b| b["key"] }
+  it "does not create .raw field for the message field" do
+    results = @es.search(:q => "message.raw:\"sample message here\"")
+    insist { results["hits"]["total"] } == 0
+  end
 
-      insist { terms }.include?("us")
+  it "creates .raw field for nested message fields" do
+    results = @es.search(:q => "somemessage.message.raw:\"sample nested message here\"")
+    insist { results["hits"]["total"] } == 1
+  end
 
-      # 'at' is a stopword, make sure stopwords are not ignored.
-      insist { terms }.include?("at")
-    end
+  it "creates .raw field from any string field which is not_analyzed" do
+    results = @es.search(:q => "country.raw:\"us\"")
+    insist { results["hits"]["total"] } == 1
+    insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+
+    # partial or terms should not work.
+    results = @es.search(:q => "country.raw:\"u\"")
+    insist { results["hits"]["total"] } == 0
+  end
+
+  it "make [geoip][location] a geo_point" do
+    expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
+  end
+
+  it "aggregate .raw results correctly " do
+    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.raw" } } } })["aggregations"]["my_agg"]
+    terms = results["buckets"].collect { |b| b["key"] }
+
+    insist { terms }.include?("us")
+
+    # 'at' is a stopword, make sure stopwords are not ignored.
+    insist { terms }.include?("at")
   end
 end
+
+