logstash-output-elasticsearch 11.3.0-java → 11.4.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -94,6 +94,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
94
94
  require "logstash/outputs/elasticsearch/ilm"
95
95
  require "logstash/outputs/elasticsearch/data_stream_support"
96
96
  require 'logstash/plugin_mixins/ecs_compatibility_support'
97
+ require 'logstash/plugin_mixins/deprecation_logger_support'
97
98
 
98
99
  # Protocol agnostic methods
99
100
  include(LogStash::PluginMixins::ElasticSearch::Common)
@@ -104,6 +105,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
104
105
  # ecs_compatibility option, provided by Logstash core or the support adapter.
105
106
  include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
106
107
 
108
+ # deprecation logger adapter for older Logstashes
109
+ include(LogStash::PluginMixins::DeprecationLoggerSupport)
110
+
107
111
  # Generic/API config options that any document indexer output needs
108
112
  include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
109
113
 
@@ -498,7 +502,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
498
502
  @default_index = "logstash-%{+yyyy.MM.dd}"
499
503
  @default_ilm_rollover_alias = "logstash"
500
504
  @default_template_name = 'logstash'
501
- when :v1
505
+ when :v1, :v8
502
506
  @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
503
507
  @default_ilm_rollover_alias = "ecs-logstash"
504
508
  @default_template_name = 'ecs-logstash'
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '11.3.0'
3
+ s.version = '11.4.0'
4
4
 
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Stores logs in Elasticsearch"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
25
25
  s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
26
26
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
27
27
  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
28
+ s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
28
29
 
29
30
  s.add_development_dependency 'logstash-codec-plain'
30
31
  s.add_development_dependency 'logstash-devutils'
@@ -59,7 +59,11 @@ module ESHelper
59
59
  end
60
60
 
61
61
  def self.es_version
62
- RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
62
+ [
63
+ nilify(RSpec.configuration.filter[:es_version]),
64
+ nilify(ENV['ES_VERSION']),
65
+ nilify(ENV['ELASTIC_STACK_VERSION']),
66
+ ].compact.first
63
67
  end
64
68
 
65
69
  RSpec::Matchers.define :have_hits do |expected|
@@ -12,6 +12,13 @@ describe "data streams", :integration => true do
12
12
 
13
13
  subject { LogStash::Outputs::ElasticSearch.new(options) }
14
14
 
15
+ # All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
16
+ # We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
17
+ let(:ecs_compatibility) { :v1 }
18
+ before(:each) do
19
+ allow( subject ).to receive(:ecs_compatibility).and_return(ecs_compatibility)
20
+ end
21
+
15
22
  before :each do
16
23
  @es = get_client
17
24
  @es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil
@@ -236,7 +236,8 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
236
236
  let (:settings) {
237
237
  {
238
238
  "ilm_enabled" => ilm_enabled,
239
- "hosts" => "#{get_host_port()}"
239
+ "hosts" => "#{get_host_port()}",
240
+ "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
240
241
  }
241
242
  }
242
243
  let (:small_max_doc_policy) { max_docs_policy(3) }
@@ -6,7 +6,8 @@ describe "Ingest pipeline execution behavior", :integration => true do
6
6
  settings = {
7
7
  "hosts" => "#{get_host_port()}",
8
8
  "pipeline" => "apache-logs",
9
- "data_stream" => 'false'
9
+ "data_stream" => 'false',
10
+ "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
10
11
  }
11
12
  next LogStash::Outputs::ElasticSearch.new(settings)
12
13
  end
@@ -45,7 +45,8 @@ describe "failures in bulk class expected behavior", :integration => true do
45
45
  "template_overwrite" => true,
46
46
  "hosts" => get_host_port(),
47
47
  "retry_max_interval" => 64,
48
- "retry_initial_interval" => 2
48
+ "retry_initial_interval" => 2,
49
+ "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
49
50
  }
50
51
  next LogStash::Outputs::ElasticSearch.new(settings)
51
52
  end
@@ -5,7 +5,7 @@ require "socket"
5
5
 
6
6
  describe "pool sniffer", :integration => true do
7
7
  let(:logger) { Cabin::Channel.get }
8
- let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
8
+ let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
9
9
  let(:es_host) { get_host_port.split(":").first }
10
10
  let(:es_port) { get_host_port.split(":").last }
11
11
  let(:es_ip) { IPSocket.getaddress(es_host) }
@@ -4,9 +4,17 @@ require "logstash/outputs/elasticsearch/data_stream_support"
4
4
  describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
5
5
 
6
6
  subject { LogStash::Outputs::ElasticSearch.new(options) }
7
+
7
8
  let(:options) { { 'hosts' => [ 'localhost:12345' ] } }
8
9
  let(:es_version) { '7.10.1' }
9
10
 
11
+ # All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
12
+ # We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
13
+ let(:ecs_compatibility) { :v1 }
14
+ before(:each) do
15
+ allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
16
+ end
17
+
10
18
  let(:do_register) { false }
11
19
  let(:stub_plugin_register!) do
12
20
  allow(subject).to receive(:last_es_version).and_return(es_version)
@@ -52,9 +60,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
52
60
  end
53
61
 
54
62
  it "warns when configuration is data-stream compliant (LS 7.x)" do
55
- expect( subject.logger ).to receive(:warn) do |msg|
56
- expect(msg).to include "Configuration is data stream compliant but due backwards compatibility Logstash 7.x"
57
- end
63
+ expect( subject.logger ).to receive(:warn).with(a_string_including "Configuration is data stream compliant but due backwards compatibility Logstash 7.x")
58
64
  change_constant :LOGSTASH_VERSION, '7.11.0' do
59
65
  expect( subject.data_stream_config? ).to be false
60
66
  end
@@ -66,6 +72,25 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
66
72
  end
67
73
  end
68
74
 
75
+ context 'ecs_compatibility disabled' do
76
+ let(:ecs_compatibility) { :disabled }
77
+
78
+ {
79
+ '7.x (pre-DS)' => '7.9.0',
80
+ '7.x (with DS)' => '7.11.0',
81
+ '8.0' => '8.0.0',
82
+ '8.x' => '8.1.2',
83
+ }.each do |ls_version_desc, ls_version|
84
+ context "on LS #{ls_version_desc}" do
85
+ around(:each) { |example| change_constant(:LOGSTASH_VERSION, ls_version, &example) }
86
+ it "does not use data-streams" do
87
+ expect( subject.logger ).to receive(:debug).with(a_string_including "ecs_compatibility is not enabled")
88
+ expect( subject.data_stream_config? ).to be false
89
+ end
90
+ end
91
+ end
92
+ end
93
+
69
94
  context 'non-compatible ES' do
70
95
 
71
96
  let(:es_version) { '7.8.0' }
@@ -95,6 +120,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
95
120
  before { allow(subject).to receive(:last_es_version).and_return(es_version) }
96
121
 
97
122
  it "does not use data-streams on LS 7.x" do
123
+ expect( subject.logger ).to receive(:warn).with(a_string_including "Logstash 7.x will not assume writing to a data-stream")
98
124
  change_constant :LOGSTASH_VERSION, '7.10.0' do
99
125
  expect( subject.data_stream_config? ).to be false
100
126
  end
@@ -197,12 +223,34 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
197
223
  end
198
224
  end
199
225
 
200
- it "does use data-streams on LS 8.0" do
226
+ it "does use data-streams on LS 8.x" do
201
227
  change_constant :LOGSTASH_VERSION, '8.1.0' do
202
228
  expect( subject.data_stream_config? ).to be true
203
229
  end
204
230
  end
205
231
 
232
+ context 'with ecs_compatibility disabled' do
233
+ let(:ecs_compatibility) { :disabled }
234
+
235
+ context 'when running on LS 7.x' do
236
+ around(:each) { |example| change_constant(:LOGSTASH_VERSION, '7.15.1', &example) }
237
+
238
+ it "emits a deprecation warning and uses data streams anway" do
239
+ expect( subject.deprecation_logger ).to receive(:deprecated).with(a_string_including "`data_stream => true` will require the plugin to be run in ECS compatibility mode")
240
+ expect( subject.data_stream_config? ).to be true
241
+ end
242
+ end
243
+
244
+ context 'when running on LS 8.x' do
245
+ around(:each) { |example| change_constant(:LOGSTASH_VERSION, '8.0.0', &example) }
246
+
247
+ it "errors helpfully" do
248
+ expect{ subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, a_string_including("Invalid data stream configuration: `ecs_compatibility => disabled`"))
249
+ end
250
+ end
251
+
252
+ end
253
+
206
254
  context 'non-compatible ES' do
207
255
 
208
256
  let(:es_version) { '6.8.11' }
@@ -10,11 +10,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
10
10
 
11
11
  it "should raise an exception if requests are issued after close" do
12
12
  subject.close
13
- expect { subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
14
- end
15
-
16
- it "should implement host unreachable exceptions" do
17
- expect(subject.host_unreachable_exceptions).to be_a(Array)
13
+ begin
14
+ subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/')
15
+ fail 'expected to raise a HostUnreachableError'
16
+ rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
17
+ expect( e.original_error ).to be_a ::Manticore::ClientStoppedException
18
+ end
18
19
  end
19
20
 
20
21
  describe "auth" do
@@ -4,7 +4,7 @@ require 'cabin'
4
4
 
5
5
  describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
6
6
  let(:logger) { Cabin::Channel.get }
7
- let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
7
+ let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
8
8
  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
9
9
  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
10
10
  let(:es_node_versions) { [ "0.0.0" ] }
@@ -4,6 +4,8 @@ require "flores/random"
4
4
  require 'concurrent/atomic/count_down_latch'
5
5
  require "logstash/outputs/elasticsearch"
6
6
 
7
+ require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
8
+
7
9
  describe LogStash::Outputs::ElasticSearch do
8
10
  subject(:elasticsearch_output_instance) { described_class.new(options) }
9
11
  let(:options) { {} }
@@ -922,6 +924,25 @@ describe LogStash::Outputs::ElasticSearch do
922
924
  end
923
925
  end
924
926
 
927
+ describe 'ECS Compatibility Support', :ecs_compatibility_support do
928
+ [
929
+ :disabled,
930
+ :v1,
931
+ :v8,
932
+ ].each do |ecs_compatibility|
933
+ context "When initialized with `ecs_compatibility => #{ecs_compatibility}`" do
934
+ let(:options) { Hash.new }
935
+ subject(:output) { described_class.new(options.merge("ecs_compatibility" => "#{ecs_compatibility}")) }
936
+ context 'when registered' do
937
+ before(:each) { output.register }
938
+ it 'has the correct effective ECS compatibility setting' do
939
+ expect(output.ecs_compatibility).to eq(ecs_compatibility)
940
+ end
941
+ end
942
+ end
943
+ end
944
+ end
945
+
925
946
  describe "post-register ES setup" do
926
947
  let(:do_register) { false }
927
948
  let(:es_version) { '7.10.0' } # DS default on LS 8.x
@@ -959,7 +980,7 @@ describe LogStash::Outputs::ElasticSearch do
959
980
  context 'error raised' do
960
981
 
961
982
  let(:es_version) { '7.8.0' }
962
- let(:options) { super().merge('data_stream' => 'true') }
983
+ let(:options) { super().merge('data_stream' => 'true', 'ecs_compatibility' => 'v1') }
963
984
  let(:latch) { Concurrent::CountDownLatch.new }
964
985
 
965
986
  before do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.3.0
4
+ version: 11.4.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-11-03 00:00:00.000000000 Z
11
+ date: 2022-01-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,20 @@ dependencies:
84
84
  - - "~>"
85
85
  - !ruby/object:Gem::Version
86
86
  version: '1.0'
87
+ - !ruby/object:Gem::Dependency
88
+ requirement: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - "~>"
91
+ - !ruby/object:Gem::Version
92
+ version: '1.0'
93
+ name: logstash-mixin-deprecation_logger_support
94
+ prerelease: false
95
+ type: :runtime
96
+ version_requirements: !ruby/object:Gem::Requirement
97
+ requirements:
98
+ - - "~>"
99
+ - !ruby/object:Gem::Version
100
+ version: '1.0'
87
101
  - !ruby/object:Gem::Dependency
88
102
  requirement: !ruby/object:Gem::Requirement
89
103
  requirements: