logstash-output-elasticsearch 10.8.2-java → 11.0.1-java

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -6,7 +6,8 @@ if ESHelper.es_version_satisfies?(">= 5")
       require "logstash/outputs/elasticsearch"
       settings = {
         "hosts" => "#{get_host_port()}",
-        "pipeline" => "apache-logs"
+        "pipeline" => "apache-logs",
+        "data_stream" => 'false'
       }
       next LogStash::Outputs::ElasticSearch.new(settings)
     end
@@ -52,9 +53,10 @@ if ESHelper.es_version_satisfies?(">= 5")
       @es.indices.refresh
 
       #Wait or fail until everything's indexed.
-      Stud::try(20.times) do
+      Stud::try(10.times) do
         r = @es.search(index: 'logstash-*')
         expect(r).to have_hits(1)
+        sleep(0.1)
       end
     end
 

data/spec/integration/outputs/retry_spec.rb

@@ -4,9 +4,21 @@ require_relative "../../../spec/es_spec_helper"
 describe "failures in bulk class expected behavior", :integration => true do
   let(:template) { '{"template" : "not important, will be updated by :index"}' }
   let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-  let(:action1) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1]) }
+  let(:action1) do
+    if ESHelper.es_version_satisfies?(">= 6", "< 7")
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1.to_hash])
+    else
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event1.to_hash])
+    end
+  end
   let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-  let(:action2) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2]) }
+  let(:action2) do
+    if ESHelper.es_version_satisfies?(">= 6", "< 7")
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2.to_hash])
+    else
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event2.to_hash])
+    end
+  end
   let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
 
   def mock_actions_with_response(*resp)

data/spec/integration/outputs/sniffer_spec.rb

@@ -1,4 +1,3 @@
-require "logstash/devutils/rspec/spec_helper"
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
 require "json"

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+require "logstash/devutils/rspec/spec_helper"
+
+unless defined?(LogStash::OSS)
+  LogStash::OSS = ENV['DISTRIBUTION'] != "default"
+end
+
+require "logstash/outputs/elasticsearch"
+
+module LogStash::Outputs::ElasticSearch::SpecHelper
+end
+
+RSpec.configure do |config|
+  config.include LogStash::Outputs::ElasticSearch::SpecHelper
+end

data/spec/unit/http_client_builder_spec.rb

@@ -40,10 +40,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
       context "when setting bulk_path" do
         let(:bulk_path) { "/meh" }
-        let(:options) { super.merge("bulk_path" => bulk_path) }
+        let(:options) { super().merge("bulk_path" => bulk_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:bulk_path]).to eq(bulk_path)
@@ -66,7 +66,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets bulk_path to path+_bulk" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:bulk_path]).to eq("#{path}/_bulk")
@@ -88,10 +88,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     describe "healthcheck_path" do
       context "when setting healthcheck_path" do
         let(:healthcheck_path) { "/meh" }
-        let(:options) { super.merge("healthcheck_path" => healthcheck_path) }
+        let(:options) { super().merge("healthcheck_path" => healthcheck_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:healthcheck_path]).to eq(healthcheck_path)
@@ -114,7 +114,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets healthcheck_path to path" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:healthcheck_path]).to eq(path)
@@ -136,10 +136,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     describe "sniffing_path" do
       context "when setting sniffing_path" do
         let(:sniffing_path) { "/meh" }
-        let(:options) { super.merge("sniffing_path" => sniffing_path) }
+        let(:options) { super().merge("sniffing_path" => sniffing_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:sniffing_path]).to eq(sniffing_path)
@@ -162,7 +162,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets sniffing_path to path+_nodes/http" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:sniffing_path]).to eq("#{path}/_nodes/http")

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -0,0 +1,542 @@
+require_relative '../../../../spec/spec_helper'
+require "logstash/outputs/elasticsearch/data_stream_support"
+
+describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
+
+  subject { LogStash::Outputs::ElasticSearch.new(options) }
+  let(:options) { { 'hosts' => [ 'localhost:12345' ] } }
+  let(:es_version) { '7.10.1' }
+
+  let(:do_register) { false }
+  let(:stub_plugin_register!) do
+    allow(subject).to receive(:last_es_version).and_return(es_version)
+
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:start)
+
+    # stub-out unrelated (finish_register) setup:
+    allow(subject).to receive(:discover_cluster_uuid)
+    allow(subject).to receive(:install_template)
+    allow(subject).to receive(:ilm_in_use?).and_return nil
+
+    # emulate 'successful' ES connection on the same thread
+    allow(subject).to receive(:after_successful_connection) { |&block| block.call }
+    allow(subject).to receive(:stop_after_successful_connection_thread)
+
+    subject.register
+
+    allow(subject.client).to receive(:maximum_seen_major_version).and_return(Integer(es_version.split('.').first))
+
+    # allow( subject.logger ).to receive(:info) do |msg|
+    #   expect(msg).to include "New Elasticsearch output"
+    # end
+  end
+
+  @@logstash_oss = LogStash::OSS
+
+  before(:each) do
+    change_constant :OSS, false, target: LogStash # assume non-OSS by default
+
+    stub_plugin_register! if do_register
+  end
+
+  after(:each) do
+    subject.close if do_register
+
+    change_constant :OSS, @@logstash_oss, target: LogStash
+  end
+
+  context "default configuration" do
+
+    let(:options) { {} }
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does not use data-streams on LS 7.x" do
+      change_constant :LOGSTASH_VERSION, '7.10.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "warns when configuration is data-stream compliant (LS 7.x)" do
+      expect( subject.logger ).to receive(:warn) do |msg|
+        expect(msg).to include "Configuration is data stream compliant but due backwards compatibility Logstash 7.x"
+      end
+      change_constant :LOGSTASH_VERSION, '7.11.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "defaults to using data-streams on LS 8.0" do
+      change_constant :LOGSTASH_VERSION, '8.0.0' do
+        expect( subject.data_stream_config? ).to be true
+      end
+    end
+
+    context 'non-compatible ES' do
+
+      let(:es_version) { '7.8.0' }
+
+      it "does not print an error (from after_successful_connection thread)" do
+        change_constant :LOGSTASH_VERSION, '7.8.1' do
+          expect( subject.logger ).to_not receive(:error)
+          expect( subject ).to receive(:finish_register).once.and_call_original
+          stub_plugin_register!
+        end
+      end
+
+    end
+
+  end
+
+  context "ds-compatible configuration" do
+
+    let(:options) do
+      {
+          'hosts' => [ 'http://127.0.0.1:12345' ],
+          'http_compression' => 'true', 'bulk_path' => '_bulk', 'timeout' => '30',
+          'user' => 'elastic', 'password' => 'ForSearch!', 'ssl' => 'false'
+      }
+    end
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does not use data-streams on LS 7.x" do
+      change_constant :LOGSTASH_VERSION, '7.10.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "defaults to using data-streams on LS 8.0" do
+      change_constant :LOGSTASH_VERSION, '8.0.1' do
+        expect( subject.data_stream_config? ).to be true
+      end
+      change_constant :LOGSTASH_VERSION, '8.1.0' do
+        expect( subject.send(:check_data_stream_config!) ).to be true
+      end
+    end
+
+    it "uses data-streams on LS 8.0 (OSS)" do
+      change_constant :LOGSTASH_VERSION, '8.0.1' do
+        change_constant :OSS, true, target: LogStash do
+          expect( subject.data_stream_config? ).to be true
+        end
+      end
+    end
+
+    context 'old ES' do
+
+      let(:es_version) { '7.8.1' }
+
+      it "prints an error (from after_successful_connection thread) on LS 8.0" do
+        change_constant :LOGSTASH_VERSION, '8.0.0' do
+          expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
+                                                           {:es_version=>"7.8.1"})
+          stub_plugin_register!
+        end
+      end
+
+    end
+
+  end
+
+  context "default (non data-stream) configuration (on 7.x)" do
+
+    let(:options) do
+      { 'data_stream_dataset' => 'test', 'data_stream_auto_routing' => 'false', 'user' => 'elastic' }
+    end
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does not default to data-streams" do
+      expect( subject.logger ).to receive(:error) do |msg|
+        expect(msg).to include "Ambiguous configuration; data stream settings are present, but data streams are not enabled"
+      end
+      change_constant :LOGSTASH_VERSION, '7.10.2' do
+        expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
+      end
+    end
+
+    context 'explicit data_stream => false' do
+
+      let(:options) { super().merge('data_stream' => 'false') }
+
+      it "raises a configuration error (due ds specific settings)" do
+        expect( subject.logger ).to receive(:error).with(/Ambiguous configuration; data stream settings must not be present when data streams is disabled/,
+                                                         {"data_stream_auto_routing"=>"false", "data_stream_dataset"=>"test"})
+        change_constant :LOGSTASH_VERSION, '7.10.2' do
+          expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
+        end
+      end
+
+    end
+
+  end
+
+  context "(explicit) ds disabled configuration" do
+
+    let(:options) { super().merge('data_stream' => false.to_s) }
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does not use data-streams on LS 7.x" do
+      change_constant :LOGSTASH_VERSION, '7.10.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "does not use data-streams on LS 8.0" do
+      change_constant :LOGSTASH_VERSION, '8.0.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "does not print a warning" do
+      expect( subject.logger ).to_not receive(:warn)
+      change_constant :LOGSTASH_VERSION, '7.10.2' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+  end
+
+  context "(explicit) ds enabled configuration" do
+
+    let(:options) { super().merge('data_stream' => true.to_s) }
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does use data-streams on LS 7.x" do
+      change_constant :LOGSTASH_VERSION, '7.9.1' do
+        expect( subject.data_stream_config? ).to be true
+      end
+    end
+
+    it "does use data-streams on LS 8.0" do
+      change_constant :LOGSTASH_VERSION, '8.1.0' do
+        expect( subject.data_stream_config? ).to be true
+      end
+    end
+
+    context 'non-compatible ES' do
+
+      let(:es_version) { '6.8.11' }
+
+      it "prints an error (from after_successful_connection thread) on LS 7.x" do
+        change_constant :LOGSTASH_VERSION, '7.12.0' do
+          expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
+                                                           {:es_version=>"6.8.11"})
+          stub_plugin_register!
+        end
+      end
+
+      it "prints an error (from after_successful_connection thread) on LS 8.0" do
+        change_constant :LOGSTASH_VERSION, '8.0.5' do
+          expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
+                                                           {:es_version=>"6.8.11"})
+          stub_plugin_register!
+        end
+      end
+
+    end
+
+  end
+
+  describe "auto routing" do
+
+    let(:options) { super().merge('data_stream' => 'true') }
+    let(:do_register) { true }
+
+    let(:event) do
+      event = LogStash::Event.new
+      event.set '[host][hostname]', 'orangutan'
+      event
+    end
+
+    context 'with data_stream.* event data' do
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'metrics'
+          event.set '[data_stream][dataset]', 'src1'
+          event.set '[data_stream][namespace]', 'test'
+        end
+      end
+
+      it 'uses event specified target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'metrics-src1-test'
+      end
+
+    end
+
+    context 'with routing turned off' do
+
+      let(:options) { super().merge('data_stream_auto_routing' => 'false') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'metrics'
+          event.set '[data_stream][dataset]', 'src1'
+          event.set '[data_stream][namespace]', 'test'
+        end
+      end
+
+      it 'uses event specified target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'logs-generic-default'
+      end
+
+    end
+
+    context 'with partial data_stream.* data' do
+
+      let(:options) { super().merge('data_stream_dataset' => 'data') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'metrics'
+          event.set '[data_stream][dataset]', 'src1'
+        end
+      end
+
+      it 'uses event specified target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'metrics-src1-default'
+      end
+
+    end
+
+    context 'with no data_stream.* fields' do
+
+      let(:options) { super().merge('data_stream_dataset' => 'stats', 'data_stream_type' => 'metrics') }
+
+      it 'uses configuration target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'metrics-stats-default'
+      end
+
+    end
+
+    context 'with default configuration' do
+
+      it 'uses default target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'logs-generic-default'
+      end
+
+    end
+
+  end
+
+  describe "field sync" do
+
+    let(:options) { super().merge('data_stream' => 'true') }
+
+    let(:do_register) { true }
+
+    let(:event) do
+      event = LogStash::Event.new
+      event.set '[host][hostname]', 'orangutan'
+      event
+    end
+
+    context 'enabled and no event data' do
+
+      let(:options) { super().merge('data_stream_sync_fields' => 'true') }
+
+      it 'fills in DS fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "default"})
+      end
+
+    end
+
+    context 'enabled and some event data' do
+
+      let(:options) { super().merge('data_stream_dataset' => 'ds1', 'data_stream_sync_fields' => 'true') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][namespace]', 'custom'
+        end
+      end
+
+      it 'fills in missing fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "ds1", "namespace" => "custom"})
+      end
+
+      it 'does not mutate data_stream hash' do
+        data_stream = event.get('data_stream')
+        data_stream_dup = data_stream.dup
+        subject.map_events([ event ])
+        expect( data_stream ).to eql data_stream_dup
+      end
+
+    end
+
+    context 'enabled with invalid data' do
+
+      let(:options) { super().merge('data_stream_sync_fields' => 'true') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream]', false
+        end
+      end
+
+      it 'overwrites invalid data_stream field' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "default"})
+      end
+
+    end
+
+    context 'enabled having invalid data with routing disabled' do
+
+      let(:options) do
+        super().merge('data_stream_sync_fields' => 'true', 'data_stream_auto_routing' => 'false', 'data_stream_namespace' => 'ns1')
+      end
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'foo'
+          event.set '[data_stream][dataset]', false
+          event.set '[data_stream][extra]', 0
+        end
+      end
+
+      it 'overwrites invalid data_stream sub-fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "ns1", "extra" => 0})
+      end
+
+    end
+
+    context 'disabled and no event data' do
+
+      let(:options) { super().merge('data_stream_dataset' => 'ds1', 'data_stream_sync_fields' => 'false') }
+
+      it 'does not fill DS fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2].keys ).to_not include 'data_stream'
+      end
+
+    end
+
+    context 'disabled and some event data' do
+
+      let(:options) { super().merge('data_stream_sync_fields' => 'false') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'logs'
+        end
+      end
+
+      it 'does not fill DS fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({ 'type' => 'logs'})
+      end
+
+    end
+
+  end
+
+  describe "validation" do
+
+    context 'with too long dataset name' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => 'x' * 120) }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with empty dataset name' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => '') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with invalid dataset char' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => 'foo/bar') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with invalid namespace char' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_namespace' => 'foo*') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with invalid "empty" namespace' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_namespace' => ' ') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with invalid type' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_type' => 'custom') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+  end
+
+  private
+
+  def change_constant(name, new_value, target: Object)
+    old_value = target.const_get name
+    begin
+      target.send :remove_const, name
+      target.const_set name, new_value
+      yield if block_given?
+    ensure
+      if block_given?
+        target.send :remove_const, name rescue nil
+        target.const_set name, old_value
+      end
+    end
+  end
+
+end