logstash-output-elasticsearch 7.4.0-java → 7.4.1-java

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -1,18 +1,19 @@
 require_relative "../../../spec/es_spec_helper"
 
-describe "Ingest pipeline execution behavior", :integration => true, :version_greater_than_equal_to_5x => true do
-  subject! do
-    require "logstash/outputs/elasticsearch"
-    settings = {
-      "hosts" => "#{get_host_port()}",
-      "pipeline" => "apache-logs"
-    }
-    next LogStash::Outputs::ElasticSearch.new(settings)
-  end
+if ESHelper.es_version_satisfies?(">= 5")
+  describe "Ingest pipeline execution behavior", :integration => true do
+    subject! do
+      require "logstash/outputs/elasticsearch"
+      settings = {
+        "hosts" => "#{get_host_port()}",
+        "pipeline" => "apache-logs"
+      }
+      next LogStash::Outputs::ElasticSearch.new(settings)
+    end
 
-  let(:ftw_client) { FTW::Agent.new }
-  let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
-  let(:apache_logs_pipeline) { '
+    let(:ftw_client) { FTW::Agent.new }
+    let(:ingest_url) { "http://#{get_host_port()}/_ingest/pipeline/apache-logs" }
+    let(:apache_logs_pipeline) { '
     {
       "description" : "Pipeline to parse Apache logs",
       "processors" : [
@@ -24,52 +25,53 @@ describe "Ingest pipeline execution behavior", :integration => true, :version_gr
         }
       ]
     }'
-  }
+    }
 
-  before :each do
-    # Delete all templates first.
-    require "elasticsearch"
+    before :each do
+      # Delete all templates first.
+      require "elasticsearch"
 
-    # Clean ES of data before we start.
-    @es = get_client
-    @es.indices.delete_template(:name => "*")
+      # Clean ES of data before we start.
+      @es = get_client
+      @es.indices.delete_template(:name => "*")
 
-    # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
+      # This can fail if there are no indexes, ignore failure.
+      @es.indices.delete(:index => "*") rescue nil
 
-    # delete existing ingest pipeline
-    req = ftw_client.delete(ingest_url)
-    ftw_client.execute(req)
+      # delete existing ingest pipeline
+      req = ftw_client.delete(ingest_url)
+      ftw_client.execute(req)
 
-    # register pipeline
-    req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
-    req.headers["Content-Type"] = "application/json"
-    ftw_client.execute(req)
+      # register pipeline
+      req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
+      req.headers["Content-Type"] = "application/json"
+      ftw_client.execute(req)
 
-    #TODO: Use esclient
-    #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
+      #TODO: Use esclient
+      #@es.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion
 
-    subject.register
-    subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
-    @es.indices.refresh
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => '183.60.215.50 - - [01/Jun/2015:18:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"')])
+      @es.indices.refresh
 
-    #Wait or fail until everything's indexed.
-    Stud::try(20.times) do
-      r = @es.search
-      insist { r["hits"]["total"] } == 1
+      #Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @es.search
+        insist { r["hits"]["total"] } == 1
+      end
     end
-  end
 
-  it "indexes using the proper pipeline" do
-    results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
-    insist { results["hits"]["total"] } == 1
-    insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
-    insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
-    insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
-    insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
-    insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
-    insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
-    insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
-    insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
+    it "indexes using the proper pipeline" do
+      results = @es.search(:index => 'logstash-*', :q => "message:\"netcat\"")
+      insist { results["hits"]["total"] } == 1
+      insist { results["hits"]["hits"][0]["_source"]["response"] } == "200"
+      insist { results["hits"]["hits"][0]["_source"]["bytes"] } == "182"
+      insist { results["hits"]["hits"][0]["_source"]["verb"] } == "GET"
+      insist { results["hits"]["hits"][0]["_source"]["request"] } == "/scripts/netcat-webserver"
+      insist { results["hits"]["hits"][0]["_source"]["auth"] } == "-"
+      insist { results["hits"]["hits"][0]["_source"]["ident"] } == "-"
+      insist { results["hits"]["hits"][0]["_source"]["clientip"] } == "183.60.215.50"
+      insist { results["hits"]["hits"][0]["_source"]["junkfieldaaaa"] } == nil
+    end
   end
 end

data/spec/integration/outputs/painless_update_spec.rb

@@ -1,148 +1,150 @@
 require_relative "../../../spec/es_spec_helper"
 
-describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless', :version_greater_than_equal_to_5x => true do
-  require "logstash/outputs/elasticsearch"
+if ESHelper.es_version_satisfies?(">= 5")
+  describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
+    require "logstash/outputs/elasticsearch"
 
-  def get_es_output( options={} )
-    settings = {
-      "manage_template" => true,
-      "index" => "logstash-update",
-      "template_overwrite" => true,
-      "hosts" => get_host_port(),
-      "action" => "update",
-      "script_lang" => "painless"
-    }
-    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
-  end
-
-  before :each do
-    @es = get_client
-    # Delete all templates first.
-    # Clean ES of data before we start.
-    @es.indices.delete_template(:name => "*")
-    # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
-    @es.index(
-      :index => 'logstash-update',
-      :type => 'logs',
-      :id => "123",
-      :body => { :message => 'Test', :counter => 1 }
-    )
-    @es.indices.refresh
-  end
-
-  context "scripted updates" do
-    it "should increment a counter with event/doc 'count' variable" do
-      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("count" => 2)])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 3
+    def get_es_output( options={} )
+      settings = {
+        "manage_template" => true,
+        "index" => "logstash-update",
+        "template_overwrite" => true,
+        "hosts" => get_host_port(),
+        "action" => "update",
+        "script_lang" => "painless"
+      }
+      LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
     end
 
-    it "should increment a counter with event/doc '[data][count]' nested variable" do
-      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 4
+    before :each do
+      @es = get_client
+      # Delete all templates first.
+      # Clean ES of data before we start.
+      @es.indices.delete_template(:name => "*")
+      # This can fail if there are no indexes, ignore failure.
+      @es.indices.delete(:index => "*") rescue nil
+      @es.index(
+        :index => 'logstash-update',
+        :type => 'logs',
+        :id => "123",
+        :body => { :message => 'Test', :counter => 1 }
+      )
+      @es.indices.refresh
     end
 
-    it "should increment a counter with event/doc 'count' variable with inline script" do
-      subject = get_es_output({
-        'document_id' => "123",
-        'script' => 'ctx._source.counter += params.event.counter',
-        'script_lang' => 'painless',
-        'script_type' => 'inline'
-      })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 4
-    end
+    context "scripted updates" do
+      it "should increment a counter with event/doc 'count' variable" do
+        subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("count" => 2)])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["counter"] } == 3
+      end
 
-    it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
-        'document_id' => "123",
-        'doc_as_upsert' => true,
-        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
-        'script_lang' => 'painless',
-        'script_type' => 'inline'
-      })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 4
-    end
+      it "should increment a counter with event/doc '[data][count]' nested variable" do
+        subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["counter"] } == 4
+      end
 
-    it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
-        'document_id' => "456",
-        'doc_as_upsert' => true,
-        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
-        'script_lang' => 'painless',
-        'script_type' => 'inline'
-      })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-      insist { r["_source"]["counter"] } == 3
-    end
+      it "should increment a counter with event/doc 'count' variable with inline script" do
+        subject = get_es_output({
+          'document_id' => "123",
+          'script' => 'ctx._source.counter += params.event.counter',
+          'script_lang' => 'painless',
+          'script_type' => 'inline'
+        })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["counter"] } == 4
+      end
 
-    it "should increment a counter with event/doc 'count' variable with indexed script" do
-      @es.put_script lang: 'painless', id: 'indexed_update', body: { script: 'ctx._source.counter += params.event.count' }
-      subject = get_es_output({
-        'document_id' => "123",
-        'script' => 'indexed_update',
-        'script_lang' => 'painless',
-        'script_type' => 'indexed'
-      })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("count" => 4 )])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 5
-    end
-  end  
-    
-  context "when update with upsert" do
-    it "should create new documents with provided upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-      insist { r["_source"]["message"] } == 'upsert message'
-    end
+      it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+        subject = get_es_output({
+          'document_id' => "123",
+          'doc_as_upsert' => true,
+          'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
+          'script_lang' => 'painless',
+          'script_type' => 'inline'
+        })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["counter"] } == 4
+      end
 
-    it "should create new documents with event/doc as upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-      insist { r["_source"]["message"] } == 'sample message here'
-    end
+      it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+        subject = get_es_output({
+          'document_id' => "456",
+          'doc_as_upsert' => true,
+          'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
+          'script_lang' => 'painless',
+          'script_type' => 'inline'
+        })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["counter"] } == 3
+      end
 
-    it "should fail on documents with event/doc as upsert at external version" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
-      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
-    end
-  end
+      it "should increment a counter with event/doc 'count' variable with indexed script" do
+        @es.put_script lang: 'painless', id: 'indexed_update', body: { script: 'ctx._source.counter += params.event.count' }
+        subject = get_es_output({
+          'document_id' => "123",
+          'script' => 'indexed_update',
+          'script_lang' => 'painless',
+          'script_type' => 'indexed'
+        })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("count" => 4 )])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["counter"] } == 5
+      end
+    end  
 
-  context "updates with scripted upsert" do
-    it "should create new documents with upsert content" do
-      subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-      insist { r["_source"]["message"] } == 'upsert message'
+    context "when update with upsert" do
+      it "should create new documents with provided upsert" do
+        subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'upsert message'
+      end
+
+      it "should create new documents with event/doc as upsert" do
+        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'sample message here'
+      end
+
+      it "should fail on documents with event/doc as upsert at external version" do
+        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
     end
 
-    it "should create new documents with event/doc as script params" do
-      subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("counter" => 1)])
-      @es.indices.refresh
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-      insist { r["_source"]["counter"] } == 1
+    context "updates with scripted upsert" do
+      it "should create new documents with upsert content" do
+        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'upsert message'
+      end
+
+      it "should create new documents with event/doc as script params" do
+        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("counter" => 1)])
+        @es.indices.refresh
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["counter"] } == 1
+      end
     end
   end
 end

data/spec/integration/outputs/parent_spec.rb

@@ -1,71 +1,73 @@
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
-shared_examples "a parent indexer" do
-    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:event_count) { 10000 + rand(500) }
-    let(:parent) { "not_implemented" }
-    let(:config) { "not_implemented" }
-    let(:default_headers) {
-      {"Content-Type" => "application/json"}
-    }
-    subject { LogStash::Outputs::ElasticSearch.new(config) }
+if ESHelper.es_version_satisfies?("<= 5")
+  shared_examples "a parent indexer" do
+      let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+      let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+      let(:event_count) { 10000 + rand(500) }
+      let(:parent) { "not_implemented" }
+      let(:config) { "not_implemented" }
+      let(:default_headers) {
+        {"Content-Type" => "application/json"}
+      }
+      subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-    before do
-      # Add mapping and a parent document
-      index_url = "http://#{get_host_port()}/#{index}"
-      ftw = FTW::Agent.new
-      mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
-      ftw.put!("#{index_url}", {:body => mapping.to_json, :headers => default_headers})
-      pdoc = { "foo" => "bar" }
-      ftw.put!("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers})
+      before do
+        # Add mapping and a parent document
+        index_url = "http://#{get_host_port()}/#{index}"
+        ftw = FTW::Agent.new
+        mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
+        ftw.put!("#{index_url}", {:body => mapping.to_json, :headers => default_headers})
+        pdoc = { "foo" => "bar" }
+        ftw.put!("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers})
 
-      subject.register
-      subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
-    end
+        subject.register
+        subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
+      end
 
 
-    it "ships events" do
-      index_url = "http://#{get_host_port()}/#{index}"
+      it "ships events" do
+        index_url = "http://#{get_host_port()}/#{index}"
 
-      ftw = FTW::Agent.new
-      ftw.post!("#{index_url}/_refresh")
+        ftw = FTW::Agent.new
+        ftw.post!("#{index_url}/_refresh")
 
-      # Wait until all events are available.
-      Stud::try(10.times) do
-        query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
-        data = ""
-        response = ftw.post!("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
-        response.read_body { |chunk| data << chunk }
-        result = LogStash::Json.load(data)
-        cur_count = result["count"]
-        insist { cur_count } == event_count
+        # Wait until all events are available.
+        Stud::try(10.times) do
+          query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
+          data = ""
+          response = ftw.post!("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
+          response.read_body { |chunk| data << chunk }
+          result = LogStash::Json.load(data)
+          cur_count = result["count"]
+          insist { cur_count } == event_count
+        end
       end
-    end
-end
+  end
 
-describe "(http protocol) index events with static parent", :integration => true do
-  it_behaves_like 'a parent indexer' do
-    let(:parent) { "test" }
-    let(:config) {
-      {
-        "hosts" => get_host_port,
-        "index" => index,
-        "parent" => parent
+  describe "(http protocol) index events with static parent", :integration => true do
+    it_behaves_like 'a parent indexer' do
+      let(:parent) { "test" }
+      let(:config) {
+        {
+          "hosts" => get_host_port,
+          "index" => index,
+          "parent" => parent
+        }
       }
-    }
+    end
   end
-end
 
-describe "(http_protocol) index events with fieldref in parent value", :integration => true do
-  it_behaves_like 'a parent indexer' do
-    let(:config) {
-      {
-        "hosts" => get_host_port,
-        "index" => index,
-        "parent" => "%{link_to}"
+  describe "(http_protocol) index events with fieldref in parent value", :integration => true do
+    it_behaves_like 'a parent indexer' do
+      let(:config) {
+        {
+          "hosts" => get_host_port,
+          "index" => index,
+          "parent" => "%{link_to}"
+        }
       }
-    }
+    end
   end
 end