logstash-output-elasticsearch 10.8.1-java → 11.0.0-java

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.8.1'
+  s.version         = '11.0.0'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -23,13 +23,13 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
-  s.add_runtime_dependency 'cabin', ['~> 0.6']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
+  s.add_development_dependency 'cabin', ['~> 0.6']
   # Still used in some specs, we should remove this ASAP
   s.add_development_dependency 'elasticsearch'
 end

data/spec/es_spec_helper.rb

@@ -1,5 +1,5 @@
-require "logstash/devutils/rspec/spec_helper"
-require 'manticore'
+require_relative './spec_helper'
+
 require 'elasticsearch'
 require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
 require_relative "support/elasticsearch/api/actions/get_alias"
@@ -8,10 +8,7 @@ require_relative "support/elasticsearch/api/actions/get_ilm_policy"
 require_relative "support/elasticsearch/api/actions/put_ilm_policy"
 
 require 'json'
-
-unless defined?(LogStash::OSS)
-  LogStash::OSS = ENV['DISTRIBUTION'] != "default"
-end
+require 'cabin'
 
 module ESHelper
   def get_host_port

data/spec/integration/outputs/data_stream_spec.rb

@@ -0,0 +1,61 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+
+describe "data streams", :integration => true do
+
+  let(:ds_name) { "logs-#{ds_dataset}-default" }
+  let(:ds_dataset) { 'integration_test' }
+
+  let(:options) do
+    { "data_stream" => 'true', "data_stream_dataset" => ds_dataset, "hosts" => get_host_port() }
+  end
+
+  subject { LogStash::Outputs::ElasticSearch.new(options) }
+
+  before :each do
+    @es = get_client
+    @es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil
+
+    es_version = @es.info['version']['number']
+    if Gem::Version.create(es_version) < Gem::Version.create('7.9.0')
+      skip "ES version #{es_version} does not support data-streams"
+    end
+  end
+
+  it "creates a new document" do
+    subject.register
+    subject.multi_receive([LogStash::Event.new("message" => "MSG 111")])
+
+    @es.indices.refresh
+
+    Stud::try(3.times) do
+      r = @es.search(index: ds_name)
+
+      expect( r['hits']['total']['value'] ).to eq 1
+      doc = r['hits']['hits'].first
+      expect( doc['_source'] ).to include "message"=>"MSG 111"
+      expect( doc['_source'] ).to include "data_stream"=>{"dataset"=>ds_dataset, "type"=>"logs", "namespace"=>"default"}
+    end
+  end
+
+  context "with document_id" do
+
+    let(:document_id) { '1234567890' }
+    let(:options) { super().merge("document_id" => document_id) }
+
+    it "creates a new document" do
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "foo")])
+
+      @es.indices.refresh
+
+      Stud::try(3.times) do
+        r = @es.search(index: ds_name, body: { query: { match: { _id: document_id } } })
+        expect( r['hits']['total']['value'] ).to eq 1
+        doc = r['hits']['hits'].first
+        expect( doc['_source'] ).to include "message"=>"foo"
+      end
+    end
+
+  end
+end

data/spec/integration/outputs/ilm_spec.rb

@@ -5,7 +5,7 @@ shared_examples_for 'an ILM enabled Logstash' do
   context 'with a policy with a maximum number of documents' do
     let (:policy) { small_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-custom"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should rollover when the policy max docs is reached' do
       put_policy(@es, ilm_policy_name, policy)
@@ -54,7 +54,7 @@ shared_examples_for 'an ILM enabled Logstash' do
   context 'with a policy where the maximum number of documents is not reached' do
     let (:policy) { large_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-custom-policy"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should ingest into a single index when max docs is not reached' do
       put_policy(@es,ilm_policy_name, policy)
@@ -119,7 +119,7 @@ shared_examples_for 'an ILM disabled Logstash' do
   context 'with an existing policy that will roll over' do
     let (:policy) { small_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-3_docs"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should not roll over indices' do
       subject.register
@@ -155,7 +155,7 @@ shared_examples_for 'an ILM disabled Logstash' do
 
   context 'with a custom template name' do
     let (:template_name) { "logstash_custom_template_name" }
-    let (:settings) { super.merge('template_name' => template_name)}
+    let (:settings) { super().merge('template_name' => template_name)}
 
     it 'should not write the ILM settings into the template' do
       subject.register
@@ -195,28 +195,32 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
   subject { LogStash::Outputs::ElasticSearch.new(settings) }
 
   context 'when ilm is enabled in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => true }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => true }) }
 
     it 'should raise a configuration error' do
+      # TODO should be refactored not to rely on plugin internals
+      finish_register = subject.method(:finish_register)
+      expect(subject).to receive(:finish_register)
       expect do
         begin
           subject.register
-          sleep(1)
+          finish_register.call
+          sleep(1.5) # wait_for_successful_connection (for the thread to raise)
         ensure
-          subject.stop_template_installer
+          subject.send :stop_after_successful_connection_thread
         end
       end.to raise_error(LogStash::ConfigurationError)
     end
   end
 
   context 'when ilm is disabled in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => false }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => false }) }
 
     it_behaves_like 'an ILM disabled Logstash'
   end
 
   context 'when ilm is set to auto in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => 'auto' }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => 'auto' }) }
 
     it_behaves_like 'an ILM disabled Logstash'
   end
@@ -286,7 +290,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when using the default policy' do
         context 'with a custom pattern' do
-          let (:settings) { super.merge("ilm_pattern" => "000001")}
+          let (:settings) { super().merge("ilm_pattern" => "000001")}
           it 'should create a rollover alias' do
             expect(@es.indices.exists_alias(name: "logstash")).to be_falsey
             subject.register
@@ -346,7 +350,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when not using the default policy' do
         let (:ilm_policy_name) {"logstash-policy-small"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
         let (:policy) { small_max_doc_policy }
 
         before do
@@ -363,7 +367,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when using a time based policy' do
         let (:ilm_policy_name) {"logstash-policy-time"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
         let (:policy) { max_age_policy("1d") }
 
         before do
@@ -409,7 +413,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
         end
 
-        let (:settings) { super.merge("template" => template,
+        let (:settings) { super().merge("template" => template,
                                       "index" => "overwrite-4")}
 
         it 'should not overwrite the index patterns' do
@@ -426,7 +430,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         let (:ilm_rollover_alias) { "logstash_the_cat_in_the_hat" }
         let (:index) { ilm_rollover_alias }
         let(:expected_index) { index }
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name,
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name,
                                       "template" => template,
                                       "ilm_rollover_alias" => ilm_rollover_alias)}
 
@@ -480,7 +484,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
         context 'with a different template_name' do
           let (:template_name) { "logstash_custom_template_name" }
-          let (:settings) { super.merge('template_name' => template_name)}
+          let (:settings) { super().merge('template_name' => template_name)}
 
           it_behaves_like 'an ILM enabled Logstash'
 
@@ -514,7 +518,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     end
 
     context 'when ilm_enabled is the default' do
-      let (:settings) { super.tap{|x|x.delete('ilm_enabled')}}
+      let (:settings) { super().tap{|x|x.delete('ilm_enabled')}}
 
       if ESHelper.es_version_satisfies?(">=7.0")
         context 'when Elasticsearch is version 7 or above' do
@@ -530,13 +534,13 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     end
 
     context 'with ilm disabled' do
-      let (:settings) { super.merge('ilm_enabled' => false )}
+      let (:settings) { super().merge('ilm_enabled' => false )}
 
       it_behaves_like 'an ILM disabled Logstash'
     end
 
     context 'with ilm disabled using a string' do
-      let (:settings) { super.merge('ilm_enabled' => 'false' )}
+      let (:settings) { super().merge('ilm_enabled' => 'false' )}
 
       it_behaves_like 'an ILM disabled Logstash'
     end

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -6,7 +6,8 @@ if ESHelper.es_version_satisfies?(">= 5")
       require "logstash/outputs/elasticsearch"
       settings = {
         "hosts" => "#{get_host_port()}",
-        "pipeline" => "apache-logs"
+        "pipeline" => "apache-logs",
+        "data_stream" => 'false'
       }
       next LogStash::Outputs::ElasticSearch.new(settings)
     end
@@ -52,9 +53,10 @@ if ESHelper.es_version_satisfies?(">= 5")
       @es.indices.refresh
 
       #Wait or fail until everything's indexed.
-      Stud::try(20.times) do
+      Stud::try(10.times) do
         r = @es.search(index: 'logstash-*')
         expect(r).to have_hits(1)
+        sleep(0.1)
       end
     end
 

data/spec/integration/outputs/retry_spec.rb

@@ -4,9 +4,21 @@ require_relative "../../../spec/es_spec_helper"
 describe "failures in bulk class expected behavior", :integration => true do
   let(:template) { '{"template" : "not important, will be updated by :index"}' }
   let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-  let(:action1) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1]) }
+  let(:action1) do
+    if ESHelper.es_version_satisfies?(">= 6", "< 7")
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1.to_hash])
+    else
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event1.to_hash])
+    end
+  end
   let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-  let(:action2) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2]) }
+  let(:action2) do
+    if ESHelper.es_version_satisfies?(">= 6", "< 7")
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2.to_hash])
+    else
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event2.to_hash])
+    end
+  end
   let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
 
   def mock_actions_with_response(*resp)

data/spec/integration/outputs/sniffer_spec.rb

@@ -1,4 +1,3 @@
-require "logstash/devutils/rspec/spec_helper"
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
 require "json"

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+require "logstash/devutils/rspec/spec_helper"
+
+unless defined?(LogStash::OSS)
+  LogStash::OSS = ENV['DISTRIBUTION'] != "default"
+end
+
+require "logstash/outputs/elasticsearch"
+
+module LogStash::Outputs::ElasticSearch::SpecHelper
+end
+
+RSpec.configure do |config|
+  config.include LogStash::Outputs::ElasticSearch::SpecHelper
+end

data/spec/unit/http_client_builder_spec.rb

@@ -40,10 +40,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
       context "when setting bulk_path" do
         let(:bulk_path) { "/meh" }
-        let(:options) { super.merge("bulk_path" => bulk_path) }
+        let(:options) { super().merge("bulk_path" => bulk_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:bulk_path]).to eq(bulk_path)
@@ -66,7 +66,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets bulk_path to path+_bulk" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:bulk_path]).to eq("#{path}/_bulk")
@@ -88,10 +88,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     describe "healthcheck_path" do
       context "when setting healthcheck_path" do
         let(:healthcheck_path) { "/meh" }
-        let(:options) { super.merge("healthcheck_path" => healthcheck_path) }
+        let(:options) { super().merge("healthcheck_path" => healthcheck_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:healthcheck_path]).to eq(healthcheck_path)
@@ -114,7 +114,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets healthcheck_path to path" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:healthcheck_path]).to eq(path)
@@ -136,10 +136,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     describe "sniffing_path" do
       context "when setting sniffing_path" do
         let(:sniffing_path) { "/meh" }
-        let(:options) { super.merge("sniffing_path" => sniffing_path) }
+        let(:options) { super().merge("sniffing_path" => sniffing_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:sniffing_path]).to eq(sniffing_path)
@@ -162,7 +162,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets sniffing_path to path+_nodes/http" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:sniffing_path]).to eq("#{path}/_nodes/http")

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -0,0 +1,542 @@
+require_relative '../../../../spec/spec_helper'
+require "logstash/outputs/elasticsearch/data_stream_support"
+
+describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
+
+  subject { LogStash::Outputs::ElasticSearch.new(options) }
+  let(:options) { { 'hosts' => [ 'localhost:12345' ] } }
+  let(:es_version) { '7.10.1' }
+
+  let(:do_register) { false }
+  let(:stub_plugin_register!) do
+    allow(subject).to receive(:last_es_version).and_return(es_version)
+
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:start)
+
+    # stub-out unrelated (finish_register) setup:
+    allow(subject).to receive(:discover_cluster_uuid)
+    allow(subject).to receive(:install_template)
+    allow(subject).to receive(:ilm_in_use?).and_return nil
+
+    # emulate 'successful' ES connection on the same thread
+    allow(subject).to receive(:after_successful_connection) { |&block| block.call }
+    allow(subject).to receive(:stop_after_successful_connection_thread)
+
+    subject.register
+
+    allow(subject.client).to receive(:maximum_seen_major_version).and_return(Integer(es_version.split('.').first))
+
+    # allow( subject.logger ).to receive(:info) do |msg|
+    #   expect(msg).to include "New Elasticsearch output"
+    # end
+  end
+
+  @@logstash_oss = LogStash::OSS
+
+  before(:each) do
+    change_constant :OSS, false, target: LogStash # assume non-OSS by default
+
+    stub_plugin_register! if do_register
+  end
+
+  after(:each) do
+    subject.close if do_register
+
+    change_constant :OSS, @@logstash_oss, target: LogStash
+  end
+
+  context "default configuration" do
+
+    let(:options) { {} }
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does not use data-streams on LS 7.x" do
+      change_constant :LOGSTASH_VERSION, '7.10.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "warns when configuration is data-stream compliant (LS 7.x)" do
+      expect( subject.logger ).to receive(:warn) do |msg|
+        expect(msg).to include "Configuration is data stream compliant but due backwards compatibility Logstash 7.x"
+      end
+      change_constant :LOGSTASH_VERSION, '7.11.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "defaults to using data-streams on LS 8.0" do
+      change_constant :LOGSTASH_VERSION, '8.0.0' do
+        expect( subject.data_stream_config? ).to be true
+      end
+    end
+
+    context 'non-compatible ES' do
+
+      let(:es_version) { '7.8.0' }
+
+      it "does not print an error (from after_successful_connection thread)" do
+        change_constant :LOGSTASH_VERSION, '7.8.1' do
+          expect( subject.logger ).to_not receive(:error)
+          expect( subject ).to receive(:finish_register).once.and_call_original
+          stub_plugin_register!
+        end
+      end
+
+    end
+
+  end
+
+  context "ds-compatible configuration" do
+
+    let(:options) do
+      {
+          'hosts' => [ 'http://127.0.0.1:12345' ],
+          'http_compression' => 'true', 'bulk_path' => '_bulk', 'timeout' => '30',
+          'user' => 'elastic', 'password' => 'ForSearch!', 'ssl' => 'false'
+      }
+    end
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does not use data-streams on LS 7.x" do
+      change_constant :LOGSTASH_VERSION, '7.10.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "defaults to using data-streams on LS 8.0" do
+      change_constant :LOGSTASH_VERSION, '8.0.1' do
+        expect( subject.data_stream_config? ).to be true
+      end
+      change_constant :LOGSTASH_VERSION, '8.1.0' do
+        expect( subject.send(:check_data_stream_config!) ).to be true
+      end
+    end
+
+    it "uses data-streams on LS 8.0 (OSS)" do
+      change_constant :LOGSTASH_VERSION, '8.0.1' do
+        change_constant :OSS, true, target: LogStash do
+          expect( subject.data_stream_config? ).to be true
+        end
+      end
+    end
+
+    context 'old ES' do
+
+      let(:es_version) { '7.8.1' }
+
+      it "prints an error (from after_successful_connection thread) on LS 8.0" do
+        change_constant :LOGSTASH_VERSION, '8.0.0' do
+          expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
+                                                           {:es_version=>"7.8.1"})
+          stub_plugin_register!
+        end
+      end
+
+    end
+
+  end
+
+  context "default (non data-stream) configuration (on 7.x)" do
+
+    let(:options) do
+      { 'data_stream_dataset' => 'test', 'data_stream_auto_routing' => 'false', 'user' => 'elastic' }
+    end
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does not default to data-streams" do
+      expect( subject.logger ).to receive(:error) do |msg|
+        expect(msg).to include "Ambiguous configuration; data stream settings are present, but data streams are not enabled"
+      end
+      change_constant :LOGSTASH_VERSION, '7.10.2' do
+        expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
+      end
+    end
+
+    context 'explicit data_stream => false' do
+
+      let(:options) { super().merge('data_stream' => 'false') }
+
+      it "raises a configuration error (due ds specific settings)" do
+        expect( subject.logger ).to receive(:error).with(/Ambiguous configuration; data stream settings must not be present when data streams is disabled/,
+                                                         {"data_stream_auto_routing"=>"false", "data_stream_dataset"=>"test"})
+        change_constant :LOGSTASH_VERSION, '7.10.2' do
+          expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
+        end
+      end
+
+    end
+
+  end
+
+  context "(explicit) ds disabled configuration" do
+
+    let(:options) { super().merge('data_stream' => false.to_s) }
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does not use data-streams on LS 7.x" do
+      change_constant :LOGSTASH_VERSION, '7.10.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "does not use data-streams on LS 8.0" do
+      change_constant :LOGSTASH_VERSION, '8.0.0' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+    it "does not print a warning" do
+      expect( subject.logger ).to_not receive(:warn)
+      change_constant :LOGSTASH_VERSION, '7.10.2' do
+        expect( subject.data_stream_config? ).to be false
+      end
+    end
+
+  end
+
+  context "(explicit) ds enabled configuration" do
+
+    let(:options) { super().merge('data_stream' => true.to_s) }
+
+    before { allow(subject).to receive(:last_es_version).and_return(es_version) }
+
+    it "does use data-streams on LS 7.x" do
+      change_constant :LOGSTASH_VERSION, '7.9.1' do
+        expect( subject.data_stream_config? ).to be true
+      end
+    end
+
+    it "does use data-streams on LS 8.0" do
+      change_constant :LOGSTASH_VERSION, '8.1.0' do
+        expect( subject.data_stream_config? ).to be true
+      end
+    end
+
+    context 'non-compatible ES' do
+
+      let(:es_version) { '6.8.11' }
+
+      it "prints an error (from after_successful_connection thread) on LS 7.x" do
+        change_constant :LOGSTASH_VERSION, '7.12.0' do
+          expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
+                                                           {:es_version=>"6.8.11"})
+          stub_plugin_register!
+        end
+      end
+
+      it "prints an error (from after_successful_connection thread) on LS 8.0" do
+        change_constant :LOGSTASH_VERSION, '8.0.5' do
+          expect( subject.logger ).to receive(:error).with(/Elasticsearch version does not support data streams/,
+                                                           {:es_version=>"6.8.11"})
+          stub_plugin_register!
+        end
+      end
+
+    end
+
+  end
+
+  describe "auto routing" do
+
+    let(:options) { super().merge('data_stream' => 'true') }
+    let(:do_register) { true }
+
+    let(:event) do
+      event = LogStash::Event.new
+      event.set '[host][hostname]', 'orangutan'
+      event
+    end
+
+    context 'with data_stream.* event data' do
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'metrics'
+          event.set '[data_stream][dataset]', 'src1'
+          event.set '[data_stream][namespace]', 'test'
+        end
+      end
+
+      it 'uses event specified target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'metrics-src1-test'
+      end
+
+    end
+
+    context 'with routing turned off' do
+
+      let(:options) { super().merge('data_stream_auto_routing' => 'false') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'metrics'
+          event.set '[data_stream][dataset]', 'src1'
+          event.set '[data_stream][namespace]', 'test'
+        end
+      end
+
+      it 'uses event specified target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'logs-generic-default'
+      end
+
+    end
+
+    context 'with partial data_stream.* data' do
+
+      let(:options) { super().merge('data_stream_dataset' => 'data') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'metrics'
+          event.set '[data_stream][dataset]', 'src1'
+        end
+      end
+
+      it 'uses event specified target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'metrics-src1-default'
+      end
+
+    end
+
+    context 'with no data_stream.* fields' do
+
+      let(:options) { super().merge('data_stream_dataset' => 'stats', 'data_stream_type' => 'metrics') }
+
+      it 'uses configuration target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'metrics-stats-default'
+      end
+
+    end
+
+    context 'with default configuration' do
+
+      it 'uses default target' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[0] ).to eql 'create'
+        expect( tuple[1] ).to include :_index => 'logs-generic-default'
+      end
+
+    end
+
+  end
+
+  describe "field sync" do
+
+    let(:options) { super().merge('data_stream' => 'true') }
+
+    let(:do_register) { true }
+
+    let(:event) do
+      event = LogStash::Event.new
+      event.set '[host][hostname]', 'orangutan'
+      event
+    end
+
+    context 'enabled and no event data' do
+
+      let(:options) { super().merge('data_stream_sync_fields' => 'true') }
+
+      it 'fills in DS fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "default"})
+      end
+
+    end
+
+    context 'enabled and some event data' do
+
+      let(:options) { super().merge('data_stream_dataset' => 'ds1', 'data_stream_sync_fields' => 'true') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][namespace]', 'custom'
+        end
+      end
+
+      it 'fills in missing fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "ds1", "namespace" => "custom"})
+      end
+
+      it 'does not mutate data_stream hash' do
+        data_stream = event.get('data_stream')
+        data_stream_dup = data_stream.dup
+        subject.map_events([ event ])
+        expect( data_stream ).to eql data_stream_dup
+      end
+
+    end
+
+    context 'enabled with invalid data' do
+
+      let(:options) { super().merge('data_stream_sync_fields' => 'true') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream]', false
+        end
+      end
+
+      it 'overwrites invalid data_stream field' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "default"})
+      end
+
+    end
+
+    context 'enabled having invalid data with routing disabled' do
+
+      let(:options) do
+        super().merge('data_stream_sync_fields' => 'true', 'data_stream_auto_routing' => 'false', 'data_stream_namespace' => 'ns1')
+      end
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'foo'
+          event.set '[data_stream][dataset]', false
+          event.set '[data_stream][extra]', 0
+        end
+      end
+
+      it 'overwrites invalid data_stream sub-fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({"type" => "logs", "dataset" => "generic", "namespace" => "ns1", "extra" => 0})
+      end
+
+    end
+
+    context 'disabled and no event data' do
+
+      let(:options) { super().merge('data_stream_dataset' => 'ds1', 'data_stream_sync_fields' => 'false') }
+
+      it 'does not fill DS fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2].keys ).to_not include 'data_stream'
+      end
+
+    end
+
+    context 'disabled and some event data' do
+
+      let(:options) { super().merge('data_stream_sync_fields' => 'false') }
+
+      let(:event) do
+        super().tap do |event|
+          event.set '[data_stream][type]', 'logs'
+        end
+      end
+
+      it 'does not fill DS fields' do
+        tuple = subject.map_events([ event ]).first
+        expect( tuple.size ).to eql 3
+        expect( tuple[2]['data_stream'] ).to eql({ 'type' => 'logs'})
+      end
+
+    end
+
+  end
+
+  describe "validation" do
+
+    context 'with too long dataset name' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => 'x' * 120) }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with empty dataset name' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => '') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with invalid dataset char' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_dataset' => 'foo/bar') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with invalid namespace char' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_namespace' => 'foo*') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with invalid "empty" namespace' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_namespace' => ' ') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+    context 'with invalid type' do
+
+      let(:options) { super().merge('data_stream' => 'true', 'data_stream_type' => 'custom') }
+
+      it 'fails' do
+        expect { LogStash::Outputs::ElasticSearch.new(options) }.to raise_error LogStash::ConfigurationError
+      end
+
+    end
+
+  end
+
+  private
+
+  def change_constant(name, new_value, target: Object)
+    old_value = target.const_get name
+    begin
+      target.send :remove_const, name
+      target.const_set name, new_value
+      yield if block_given?
+    ensure
+      if block_given?
+        target.send :remove_const, name rescue nil
+        target.const_set name, old_value
+      end
+    end
+  end
+
+end