logstash-output-elasticsearch 8.1.1-java → 8.2.0-java

data/spec/integration/outputs/sniffer_spec.rb

@@ -38,7 +38,10 @@ describe "pool sniffer", :integration => true do
   end
   
   # We do a more thorough check on these versions because we can more reliably guess the ip
-  describe("Complex sniff parsing ES 6x/5x/2x", :version_greater_than_equal_to_2x => true) do
-    include_examples("sniff parsing", true)
+
+  if ESHelper.es_version_satisfies?(">= 2")
+    describe("Complex sniff parsing ES 6x/5x/2x") do
+      include_examples("sniff parsing", true)
+    end
   end
 end

data/spec/integration/outputs/templates_5x_spec.rb

@@ -1,99 +1,98 @@
 require_relative "../../../spec/es_spec_helper"
 
-# This file is a copy of template test for 2.x. We can DRY this up later.
-describe "index template expected behavior for 5.x", :integration => true, :version_greater_than_equal_to_5x => true do
-  subject! do
-    require "logstash/outputs/elasticsearch"
-    settings = {
-      "manage_template" => true,
-      "template_overwrite" => true,
-      "hosts" => "#{get_host_port()}"
-    }
-    next LogStash::Outputs::ElasticSearch.new(settings)
-  end
-
-  before :each do
-    # Delete all templates first.
-    require "elasticsearch"
-
-    # Clean ES of data before we start.
-    @es = get_client
-    @es.indices.delete_template(:name => "*")
-
-    # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
-
-    subject.register
-
-    subject.multi_receive([
-      LogStash::Event.new("message" => "sample message here"),
-      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-      LogStash::Event.new("somevalue" => 100),
-      LogStash::Event.new("somevalue" => 10),
-      LogStash::Event.new("somevalue" => 1),
-      LogStash::Event.new("country" => "us"),
-      LogStash::Event.new("country" => "at"),
-      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-    ])
-
-    @es.indices.refresh
+if ESHelper.es_version_satisfies?(">= 5")
+  describe "index template expected behavior for 5.x", :integration => true do
+    subject! do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "manage_template" => true,
+        "template_overwrite" => true,
+        "hosts" => "#{get_host_port()}"
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
 
-    # Wait or fail until everything's indexed.
-    Stud::try(20.times) do
-      r = @es.search
-      insist { r["hits"]["total"] } == 8
+    before :each do
+      # Delete all templates first.
+      require "elasticsearch"
+
+      # Clean ES of data before we start.
+      @es = get_client
+      @es.indices.delete_template(:name => "*")
+
+      # This can fail if there are no indexes, ignore failure.
+      @es.indices.delete(:index => "*") rescue nil
+
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @es.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 8
+      end
     end
-  end
 
-  it "permits phrase searching on string fields" do
-    results = @es.search(:q => "message:\"sample message\"")
-    insist { results["hits"]["total"] } == 1
-    insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
-  end
+    it "permits phrase searching on string fields" do
+      results = @es.search(:q => "message:\"sample message\"")
+      insist { results["hits"]["total"] } == 1
+      insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+    end
 
-  it "numbers dynamically map to a numeric type and permit range queries" do
-    results = @es.search(:q => "somevalue:[5 TO 105]")
-    insist { results["hits"]["total"] } == 2
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @es.search(:q => "somevalue:[5 TO 105]")
+      insist { results["hits"]["total"] } == 2
 
-    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-    insist { values }.include?(10)
-    insist { values }.include?(100)
-    reject { values }.include?(1)
-  end
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      insist { values }.include?(10)
+      insist { values }.include?(100)
+      reject { values }.include?(1)
+    end
 
-  it "does not create .keyword field for top-level message field" do
-    results = @es.search(:q => "message.keyword:\"sample message here\"")
-    insist { results["hits"]["total"] } == 0
-  end
+    it "does not create .keyword field for top-level message field" do
+      results = @es.search(:q => "message.keyword:\"sample message here\"")
+      insist { results["hits"]["total"] } == 0
+    end
 
-  it "creates .keyword field for nested message fields" do
-    results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
-    insist { results["hits"]["total"] } == 1
-  end
+    it "creates .keyword field for nested message fields" do
+      results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+      insist { results["hits"]["total"] } == 1
+    end
 
-  it "creates .keyword field from any string field which is not_analyzed" do
-    results = @es.search(:q => "country.keyword:\"us\"")
-    insist { results["hits"]["total"] } == 1
-    insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+    it "creates .keyword field from any string field which is not_analyzed" do
+      results = @es.search(:q => "country.keyword:\"us\"")
+      insist { results["hits"]["total"] } == 1
+      insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
 
-    # partial or terms should not work.
-    results = @es.search(:q => "country.keyword:\"u\"")
-    insist { results["hits"]["total"] } == 0
-  end
+      # partial or terms should not work.
+      results = @es.search(:q => "country.keyword:\"u\"")
+      insist { results["hits"]["total"] } == 0
+    end
 
-  it "make [geoip][location] a geo_point" do
-    expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
-  end
+    it "make [geoip][location] a geo_point" do
+      expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
+    end
 
-  it "aggregate .keyword results correctly " do
-    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
-    terms = results["buckets"].collect { |b| b["key"] }
+    it "aggregate .keyword results correctly " do
+      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
 
-    insist { terms }.include?("us")
+      insist { terms }.include?("us")
 
-    # 'at' is a stopword, make sure stopwords are not ignored.
-    insist { terms }.include?("at")
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      insist { terms }.include?("at")
+    end
   end
 end
-
-

data/spec/integration/outputs/templates_spec.rb

@@ -1,98 +1,98 @@
 require_relative "../../../spec/es_spec_helper"
 
-describe "index template expected behavior", :integration => true, :version_less_than_5x => true do
-  subject! do
-    require "logstash/outputs/elasticsearch"
-    settings = {
-      "manage_template" => true,
-      "template_overwrite" => true,
-      "hosts" => "#{get_host_port()}"
-    }
-    next LogStash::Outputs::ElasticSearch.new(settings)
-  end
-
-  before :each do
-    # Delete all templates first.
-    require "elasticsearch"
-
-    # Clean ES of data before we start.
-    @es = get_client
-    @es.indices.delete_template(:name => "*")
-
-    # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
-
-    subject.register
-
-    subject.multi_receive([
-      LogStash::Event.new("message" => "sample message here"),
-      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-      LogStash::Event.new("somevalue" => 100),
-      LogStash::Event.new("somevalue" => 10),
-      LogStash::Event.new("somevalue" => 1),
-      LogStash::Event.new("country" => "us"),
-      LogStash::Event.new("country" => "at"),
-      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-    ])
-
-    @es.indices.refresh
+if ESHelper.es_version_satisfies?("< 5")
+  describe "index template expected behavior", :integration => true do
+    subject! do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "manage_template" => true,
+        "template_overwrite" => true,
+        "hosts" => "#{get_host_port()}"
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
 
-    # Wait or fail until everything's indexed.
-    Stud::try(20.times) do
-      r = @es.search
-      insist { r["hits"]["total"] } == 8
+    before :each do
+      # Delete all templates first.
+      require "elasticsearch"
+
+      # Clean ES of data before we start.
+      @es = get_client
+      @es.indices.delete_template(:name => "*")
+
+      # This can fail if there are no indexes, ignore failure.
+      @es.indices.delete(:index => "*") rescue nil
+
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @es.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 8
+      end
     end
-  end
 
-  it "permits phrase searching on string fields" do
-    results = @es.search(:q => "message:\"sample message\"")
-    insist { results["hits"]["total"] } == 1
-    insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
-  end
+    it "permits phrase searching on string fields" do
+      results = @es.search(:q => "message:\"sample message\"")
+      insist { results["hits"]["total"] } == 1
+      insist { results["hits"]["hits"][0]["_source"]["message"] } == "sample message here"
+    end
 
-  it "numbers dynamically map to a numeric type and permit range queries" do
-    results = @es.search(:q => "somevalue:[5 TO 105]")
-    insist { results["hits"]["total"] } == 2
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @es.search(:q => "somevalue:[5 TO 105]")
+      insist { results["hits"]["total"] } == 2
 
-    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-    insist { values }.include?(10)
-    insist { values }.include?(100)
-    reject { values }.include?(1)
-  end
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      insist { values }.include?(10)
+      insist { values }.include?(100)
+      reject { values }.include?(1)
+    end
 
-  it "does not create .raw field for the message field" do
-    results = @es.search(:q => "message.raw:\"sample message here\"")
-    insist { results["hits"]["total"] } == 0
-  end
+    it "does not create .raw field for the message field" do
+      results = @es.search(:q => "message.raw:\"sample message here\"")
+      insist { results["hits"]["total"] } == 0
+    end
 
-  it "creates .raw field for nested message fields" do
-    results = @es.search(:q => "somemessage.message.raw:\"sample nested message here\"")
-    insist { results["hits"]["total"] } == 1
-  end
+    it "creates .raw field for nested message fields" do
+      results = @es.search(:q => "somemessage.message.raw:\"sample nested message here\"")
+      insist { results["hits"]["total"] } == 1
+    end
 
-  it "creates .raw field from any string field which is not_analyzed" do
-    results = @es.search(:q => "country.raw:\"us\"")
-    insist { results["hits"]["total"] } == 1
-    insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
+    it "creates .raw field from any string field which is not_analyzed" do
+      results = @es.search(:q => "country.raw:\"us\"")
+      insist { results["hits"]["total"] } == 1
+      insist { results["hits"]["hits"][0]["_source"]["country"] } == "us"
 
-    # partial or terms should not work.
-    results = @es.search(:q => "country.raw:\"u\"")
-    insist { results["hits"]["total"] } == 0
-  end
+      # partial or terms should not work.
+      results = @es.search(:q => "country.raw:\"u\"")
+      insist { results["hits"]["total"] } == 0
+    end
 
-  it "make [geoip][location] a geo_point" do
-    expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
-  end
+    it "make [geoip][location] a geo_point" do
+      expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
+    end
 
-  it "aggregate .raw results correctly " do
-    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.raw" } } } })["aggregations"]["my_agg"]
-    terms = results["buckets"].collect { |b| b["key"] }
+    it "aggregate .raw results correctly " do
+      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.raw" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
 
-    insist { terms }.include?("us")
+      insist { terms }.include?("us")
 
-    # 'at' is a stopword, make sure stopwords are not ignored.
-    insist { terms }.include?("at")
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      insist { terms }.include?("at")
+    end
   end
 end
-
-

data/spec/integration/outputs/update_spec.rb

@@ -1,114 +1,116 @@
 require_relative "../../../spec/es_spec_helper"
 
-describe "Update actions without scripts", :integration => true, :version_greater_than_equal_to_2x => true do
-  require "logstash/outputs/elasticsearch"
-
-  def get_es_output( options={} )
-    settings = {
-      "manage_template" => true,
-      "index" => "logstash-update",
-      "template_overwrite" => true,
-      "hosts" => get_host_port(),
-      "action" => "update"
-    }
-    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
-  end
-
-  before :each do
-    @es = get_client
-    # Delete all templates first.
-    # Clean ES of data before we start.
-    @es.indices.delete_template(:name => "*")
-    # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
-    @es.index(
-      :index => 'logstash-update',
-      :type => 'logs',
-      :id => "123",
-      :body => { :message => 'Test', :counter => 1 }
-    )
-    @es.indices.refresh
-  end
-
-  it "should fail without a document_id" do
-    subject = get_es_output
-    expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-  end
-
-  context "when update only" do
-    it "should not create new document" do
-      subject = get_es_output({ 'document_id' => "456" } )
-      subject.register
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
-    end
-
-    it "should update existing document" do
-      subject = get_es_output({ 'document_id' => "123" })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["message"] } == 'updated message here'
-    end
-
-    # The es ruby client treats the data field differently. Make sure this doesn't
-    # raise an exception
-    it "should update an existing document that has a 'data' field" do
-      subject = get_es_output({ 'document_id' => "123" })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["data"] } == 'updated message here'
-      insist { r["_source"]["message"] } == 'foo'
-    end
-
-    it "should allow default (internal) version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99" })
-      subject.register
+if ESHelper.es_version_satisfies?(">= 2")
+  describe "Update actions without scripts", :integration => true do
+    require "logstash/outputs/elasticsearch"
+
+    def get_es_output( options={} )
+      settings = {
+        "manage_template" => true,
+        "index" => "logstash-update",
+        "template_overwrite" => true,
+        "hosts" => get_host_port(),
+        "action" => "update"
+      }
+      LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
     end
 
-    it "should allow internal version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
-      subject.register
+    before :each do
+      @es = get_client
+      # Delete all templates first.
+      # Clean ES of data before we start.
+      @es.indices.delete_template(:name => "*")
+      # This can fail if there are no indexes, ignore failure.
+      @es.indices.delete(:index => "*") rescue nil
+      @es.index(
+        :index => 'logstash-update',
+        :type => 'logs',
+        :id => "123",
+        :body => { :message => 'Test', :counter => 1 }
+      )
+      @es.indices.refresh
     end
 
-    it "should not allow external version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
+    it "should fail without a document_id" do
+      subject = get_es_output
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
-    it "should not allow external_gt version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
-      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-    end
+    context "when update only" do
+      it "should not create new document" do
+        subject = get_es_output({ 'document_id' => "456" } )
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+
+      it "should update existing document" do
+        subject = get_es_output({ 'document_id' => "123" })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["message"] } == 'updated message here'
+      end
+
+      # The es ruby client treats the data field differently. Make sure this doesn't
+      # raise an exception
+      it "should update an existing document that has a 'data' field" do
+        subject = get_es_output({ 'document_id' => "123" })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["data"] } == 'updated message here'
+        insist { r["_source"]["message"] } == 'foo'
+      end
+
+      it "should allow default (internal) version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+        subject.register
+      end
+
+      it "should allow internal version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+        subject.register
+      end
+
+      it "should not allow external version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+
+      it "should not allow external_gt version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+
+      it "should not allow external_gte version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
 
-    it "should not allow external_gte version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
-      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
-  end
-
-  context "when update with upsert" do
-    it "should create new documents with provided upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-      insist { r["_source"]["message"] } == 'upsert message'
-    end
-
-    it "should create new documents with event/doc as upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-      insist { r["_source"]["message"] } == 'sample message here'
-    end
-
-    it "should fail on documents with event/doc as upsert at external version" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
-      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    context "when update with upsert" do
+      it "should create new documents with provided upsert" do
+        subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'upsert message'
+      end
+
+      it "should create new documents with event/doc as upsert" do
+        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'sample message here'
+      end
+
+      it "should fail on documents with event/doc as upsert at external version" do
+        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
     end
   end
 end