logstash-output-elasticsearch 11.12.2-java → 11.12.3-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a71925d993ccbf151c0cb88f41641e76926df0a2a398914dbe5d4ce54224c8f5
|
4
|
+
data.tar.gz: 97b17aad17959c0319ce94916570ce12f9ef67b7a22d583221ac26688e7cbe76
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '0817002e4b468d47ea5540e8a9ed63efbee2506350da2f08a3171d0e26037a4e99d5bff613f6f6fca91ea441c88ae08602b211a656e10f13f5b9a91895838bab'
|
7
|
+
data.tar.gz: 60312f326a381d28b895c8f1f7f8fae611d7fdb1ccf4f1782da15949eb84c2fb93411e998f727ba91a922052cfe785bb9a3227aa798c7e1434eeca8369024471
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 11.12.3
|
2
|
+
- Changed the log messages for data stream checks [#1109](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1109)
|
3
|
+
- Added more details about incompatible data streams supplied configurations
|
4
|
+
- Changed the data stream auto-configuration log levels from `debug` to `info`
|
5
|
+
|
1
6
|
## 11.12.2
|
2
7
|
- [Doc] Fixes the broken apache http client link [#1101](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1101)
|
3
8
|
|
@@ -28,7 +28,6 @@ module LogStash module Outputs class ElasticSearch
|
|
28
28
|
base.extend(Validator)
|
29
29
|
end
|
30
30
|
|
31
|
-
# @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
|
32
31
|
def data_stream_config?
|
33
32
|
@data_stream_config.nil? ? @data_stream_config = check_data_stream_config! : @data_stream_config
|
34
33
|
end
|
@@ -45,53 +44,64 @@ module LogStash module Outputs class ElasticSearch
|
|
45
44
|
"#{type}-#{dataset}-#{namespace}"
|
46
45
|
end
|
47
46
|
|
48
|
-
|
47
|
+
DATA_STREAMS_AND_ECS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
|
49
48
|
|
50
49
|
# @param params the user configuration for the ES output
|
51
50
|
# @note LS initialized configuration (with filled defaults) won't detect as data-stream
|
52
51
|
# compatible, only explicit (`original_params`) config should be tested.
|
53
|
-
# @return [
|
52
|
+
# @return [Boolean] whether given configuration is data-stream compatible
|
54
53
|
def check_data_stream_config!(params = original_params)
|
55
|
-
data_stream_params = params.select { |name, _| name.start_with?('data_stream_') } # exclude data_stream =>
|
56
|
-
invalid_data_stream_params = invalid_data_stream_params(params)
|
57
|
-
|
58
54
|
case data_stream_explicit_value
|
59
55
|
when false
|
60
|
-
|
61
|
-
@logger.error "Ambiguous configuration; data stream settings must not be present when data streams is disabled (caused by: `data_stream => false`)", data_stream_params
|
62
|
-
raise LogStash::ConfigurationError, "Ambiguous configuration, please remove data stream specific settings: #{data_stream_params.keys}"
|
63
|
-
end
|
56
|
+
check_disabled_data_stream_config!(params)
|
64
57
|
return false
|
65
58
|
when true
|
66
|
-
|
67
|
-
@logger.error "Invalid data stream configuration, following parameters are not supported:", invalid_data_stream_params
|
68
|
-
raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
|
69
|
-
end
|
70
|
-
if ecs_compatibility == :disabled
|
71
|
-
if ::Gem::Version.create(LOGSTASH_VERSION) < ::Gem::Version.create(DATA_STREAMS_REQUIRES_ECS_LS_VERSION)
|
72
|
-
@deprecation_logger.deprecated "In a future release of Logstash, the Elasticsearch output plugin's `data_stream => true` will require the plugin to be run in ECS compatibility mode. " + ENABLING_ECS_GUIDANCE
|
73
|
-
else
|
74
|
-
@logger.error "Invalid data stream configuration; `ecs_compatibility` must not be `disabled`. " + ENABLING_ECS_GUIDANCE
|
75
|
-
raise LogStash::ConfigurationError, "Invalid data stream configuration: `ecs_compatibility => disabled`"
|
76
|
-
end
|
77
|
-
end
|
59
|
+
check_enabled_data_stream_config!(params)
|
78
60
|
return true
|
79
|
-
else
|
80
|
-
use_data_stream = data_stream_default(
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
61
|
+
else # data_stream => auto or not set
|
62
|
+
use_data_stream = data_stream_default(params)
|
63
|
+
|
64
|
+
check_disabled_data_stream_config!(params) unless use_data_stream
|
65
|
+
|
66
|
+
@logger.info("Data streams auto configuration (`data_stream => auto` or unset) resolved to `#{use_data_stream}`")
|
67
|
+
return use_data_stream
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def check_enabled_data_stream_config!(params)
|
72
|
+
invalid_data_stream_params = invalid_data_stream_params(params)
|
73
|
+
|
74
|
+
if invalid_data_stream_params.any?
|
75
|
+
@logger.error "Invalid data stream configuration, the following parameters are not supported:", invalid_data_stream_params
|
76
|
+
raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
|
77
|
+
end
|
78
|
+
|
79
|
+
if ecs_compatibility == :disabled
|
80
|
+
if ecs_compatibility_required?
|
81
|
+
@logger.error "Invalid data stream configuration; `ecs_compatibility` must not be `disabled`. " + ENABLING_ECS_GUIDANCE
|
82
|
+
raise LogStash::ConfigurationError, "Invalid data stream configuration: `ecs_compatibility => disabled`"
|
90
83
|
end
|
91
|
-
|
84
|
+
|
85
|
+
@deprecation_logger.deprecated "In a future release of Logstash, the Elasticsearch output plugin's `data_stream => true` will require the plugin to be run in ECS compatibility mode. " + ENABLING_ECS_GUIDANCE
|
92
86
|
end
|
93
87
|
end
|
94
88
|
|
89
|
+
def check_disabled_data_stream_config!(params)
|
90
|
+
data_stream_params = data_stream_params(params)
|
91
|
+
|
92
|
+
if data_stream_params.any?
|
93
|
+
@logger.error "Ambiguous configuration; data stream settings must not be present when data streams are disabled (caused by `data_stream => false`, `data_stream => auto` or unset resolved to false). " \
|
94
|
+
"You can either manually set `data_stream => true` or remove the following specific data stream settings: ", data_stream_params
|
95
|
+
|
96
|
+
raise LogStash::ConfigurationError,
|
97
|
+
"Ambiguous configuration; data stream settings must not be present when data streams are disabled: #{data_stream_params.keys}"
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def data_stream_params(params)
|
102
|
+
params.select { |name, _| name.start_with?('data_stream_') }
|
103
|
+
end
|
104
|
+
|
95
105
|
def data_stream_explicit_value
|
96
106
|
case @data_stream
|
97
107
|
when 'true'
|
@@ -131,6 +141,7 @@ module LogStash module Outputs class ElasticSearch
|
|
131
141
|
|
132
142
|
DATA_STREAMS_ORIGIN_ES_VERSION = '7.9.0'
|
133
143
|
|
144
|
+
# @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
|
134
145
|
# @return [Gem::Version] if ES supports DS nil (or raise) otherwise
|
135
146
|
def assert_es_version_supports_data_streams
|
136
147
|
fail 'no last_es_version' unless last_es_version # assert - should not happen
|
@@ -144,31 +155,26 @@ module LogStash module Outputs class ElasticSearch
|
|
144
155
|
es_version # return truthy
|
145
156
|
end
|
146
157
|
|
147
|
-
DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
|
148
|
-
|
149
158
|
# when data_stream => is either 'auto' or not set
|
150
|
-
|
151
|
-
# @param invalid_data_stream_config [#any?#inspect]
|
152
|
-
def data_stream_default(data_stream_params, invalid_data_stream_config)
|
159
|
+
def data_stream_default(params)
|
153
160
|
if ecs_compatibility == :disabled
|
154
|
-
@logger.
|
161
|
+
@logger.info("Not eligible for data streams because ecs_compatibility is not enabled. " + ENABLING_ECS_GUIDANCE)
|
155
162
|
return false
|
156
163
|
end
|
157
164
|
|
158
|
-
|
165
|
+
invalid_data_stream_params = invalid_data_stream_params(params)
|
159
166
|
|
160
|
-
if
|
161
|
-
if
|
162
|
-
@logger.
|
167
|
+
if data_stream_and_ecs_enabled_by_default?
|
168
|
+
if invalid_data_stream_params.any?
|
169
|
+
@logger.info("Not eligible for data streams because config contains one or more settings that are not compatible with data streams: #{invalid_data_stream_params.inspect}")
|
163
170
|
return false
|
164
171
|
end
|
165
172
|
|
166
|
-
@logger.debug 'Configuration is data stream compliant'
|
167
173
|
return true
|
168
174
|
end
|
169
175
|
|
170
176
|
# LS 7.x
|
171
|
-
if !
|
177
|
+
if !invalid_data_stream_params.any? && !data_stream_params(params).any?
|
172
178
|
@logger.warn "Configuration is data stream compliant but due backwards compatibility Logstash 7.x will not assume " +
|
173
179
|
"writing to a data-stream, default behavior will change on Logstash 8.0 " +
|
174
180
|
"(set `data_stream => true/false` to disable this warning)"
|
@@ -176,6 +182,14 @@ module LogStash module Outputs class ElasticSearch
|
|
176
182
|
false
|
177
183
|
end
|
178
184
|
|
185
|
+
def ecs_compatibility_required?
|
186
|
+
data_stream_and_ecs_enabled_by_default?
|
187
|
+
end
|
188
|
+
|
189
|
+
def data_stream_and_ecs_enabled_by_default?
|
190
|
+
::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create(DATA_STREAMS_AND_ECS_ENABLED_BY_DEFAULT_LS_VERSION)
|
191
|
+
end
|
192
|
+
|
179
193
|
# an {event_action_tuple} replacement when a data-stream configuration is detected
|
180
194
|
def data_stream_event_action_tuple(event)
|
181
195
|
event_data = event.to_hash
|
@@ -300,6 +300,10 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
300
300
|
# to build_client down to the Pool class.
|
301
301
|
@client = build_client(LicenseChecker.new(@logger))
|
302
302
|
|
303
|
+
# Avoids race conditions in the @data_stream_config initialization (invoking check_data_stream_config! twice).
|
304
|
+
# It's being concurrently invoked by this register method and by the finish_register on the @after_successful_connection_thread
|
305
|
+
data_stream_enabled = data_stream_config?
|
306
|
+
|
303
307
|
@after_successful_connection_thread = after_successful_connection do
|
304
308
|
begin
|
305
309
|
finish_register
|
@@ -324,7 +328,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
324
328
|
raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
|
325
329
|
end
|
326
330
|
|
327
|
-
if
|
331
|
+
if data_stream_enabled
|
328
332
|
@event_mapper = -> (e) { data_stream_event_action_tuple(e) }
|
329
333
|
@event_target = -> (e) { data_stream_name(e) }
|
330
334
|
@index = "#{data_stream_type}-#{data_stream_dataset}-#{data_stream_namespace}".freeze # default name
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-elasticsearch'
|
3
|
-
s.version = '11.12.
|
3
|
+
s.version = '11.12.3'
|
4
4
|
s.licenses = ['apache-2.0']
|
5
5
|
s.summary = "Stores logs in Elasticsearch"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -84,7 +84,8 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
84
84
|
context "on LS #{ls_version_desc}" do
|
85
85
|
around(:each) { |example| change_constant(:LOGSTASH_VERSION, ls_version, &example) }
|
86
86
|
it "does not use data-streams" do
|
87
|
-
expect( subject.logger ).to receive(:
|
87
|
+
expect( subject.logger ).to receive(:info).with(a_string_including "ecs_compatibility is not enabled")
|
88
|
+
expect( subject.logger ).to receive(:info).with(a_string_including "Data streams auto configuration (`data_stream => auto` or unset) resolved to `false`")
|
88
89
|
expect( subject.data_stream_config? ).to be false
|
89
90
|
end
|
90
91
|
end
|
@@ -161,7 +162,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
161
162
|
|
162
163
|
it "does not default to data-streams" do
|
163
164
|
expect( subject.logger ).to receive(:error) do |msg|
|
164
|
-
expect(msg).to include "Ambiguous configuration; data stream settings
|
165
|
+
expect(msg).to include "Ambiguous configuration; data stream settings must not be present when data streams are disabled"
|
165
166
|
end
|
166
167
|
change_constant :LOGSTASH_VERSION, '7.10.2' do
|
167
168
|
expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
|
@@ -173,8 +174,8 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
|
|
173
174
|
let(:options) { super().merge('data_stream' => 'false') }
|
174
175
|
|
175
176
|
it "raises a configuration error (due ds specific settings)" do
|
176
|
-
expect( subject.logger ).to receive(:error).with(/Ambiguous configuration; data stream settings must not be present when data streams
|
177
|
-
|
177
|
+
expect( subject.logger ).to receive(:error).with(/Ambiguous configuration; data stream settings must not be present when data streams are disabled/,
|
178
|
+
{"data_stream_auto_routing"=>"false", "data_stream_dataset"=>"test"})
|
178
179
|
change_constant :LOGSTASH_VERSION, '7.10.2' do
|
179
180
|
expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
|
180
181
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 11.12.
|
4
|
+
version: 11.12.3
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-01-
|
11
|
+
date: 2023-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|