logstash-output-elasticsearch 11.12.4-java → 11.15.9-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +58 -0
- data/docs/index.asciidoc +214 -66
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +14 -4
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +46 -19
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +26 -3
- data/lib/logstash/outputs/elasticsearch.rb +89 -39
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +60 -8
- data/lib/logstash/plugin_mixins/elasticsearch/common.rb +42 -18
- data/logstash-output-elasticsearch.gemspec +3 -2
- data/spec/es_spec_helper.rb +12 -7
- data/spec/fixtures/test_certs/ca.crt +13 -13
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -1
- data/spec/fixtures/test_certs/test.crt +14 -14
- data/spec/fixtures/test_certs/test.der.sha256 +1 -1
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/integration/outputs/index_spec.rb +16 -16
- data/spec/integration/outputs/templates_spec.rb +11 -9
- data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +4 -2
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +9 -2
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +72 -20
- data/spec/unit/outputs/elasticsearch_spec.rb +358 -28
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +166 -50
- metadata +18 -4
@@ -96,10 +96,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
96
96
|
require "logstash/outputs/elasticsearch/data_stream_support"
|
97
97
|
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
98
98
|
require 'logstash/plugin_mixins/deprecation_logger_support'
|
99
|
+
require 'logstash/plugin_mixins/normalize_config_support'
|
99
100
|
|
100
101
|
# Protocol agnostic methods
|
101
102
|
include(LogStash::PluginMixins::ElasticSearch::Common)
|
102
103
|
|
104
|
+
# Config normalization helpers
|
105
|
+
include(LogStash::PluginMixins::NormalizeConfigSupport)
|
106
|
+
|
103
107
|
# Methods for ILM support
|
104
108
|
include(LogStash::Outputs::ElasticSearch::Ilm)
|
105
109
|
|
@@ -263,14 +267,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
263
267
|
# ILM policy to use, if undefined the default policy will be used.
|
264
268
|
config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
|
265
269
|
|
266
|
-
# List extra HTTP's error codes that are considered valid to move the events into the dead letter queue.
|
267
|
-
# It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
|
268
|
-
# The option accepts a list of natural numbers corresponding to HTTP errors codes.
|
269
|
-
config :dlq_custom_codes, :validate => :number, :list => true, :default => []
|
270
|
-
|
271
|
-
# if enabled, failed index name interpolation events go into dead letter queue.
|
272
|
-
config :dlq_on_failed_indexname_interpolation, :validate => :boolean, :default => true
|
273
|
-
|
274
270
|
attr_reader :client
|
275
271
|
attr_reader :default_index
|
276
272
|
attr_reader :default_ilm_rollover_alias
|
@@ -279,6 +275,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
279
275
|
def initialize(*params)
|
280
276
|
super
|
281
277
|
setup_ecs_compatibility_related_defaults
|
278
|
+
setup_ssl_params!
|
282
279
|
end
|
283
280
|
|
284
281
|
def register
|
@@ -307,6 +304,22 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
307
304
|
data_stream_enabled = data_stream_config?
|
308
305
|
|
309
306
|
setup_template_manager_defaults(data_stream_enabled)
|
307
|
+
# To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
|
308
|
+
@dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
|
309
|
+
|
310
|
+
@dlq_codes = DOC_DLQ_CODES.to_set
|
311
|
+
|
312
|
+
if dlq_enabled?
|
313
|
+
check_dlq_custom_codes
|
314
|
+
@dlq_codes.merge(dlq_custom_codes)
|
315
|
+
else
|
316
|
+
raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
|
317
|
+
end
|
318
|
+
|
319
|
+
setup_mapper_and_target(data_stream_enabled)
|
320
|
+
|
321
|
+
@bulk_request_metrics = metric.namespace(:bulk_requests)
|
322
|
+
@document_level_metrics = metric.namespace(:documents)
|
310
323
|
|
311
324
|
@after_successful_connection_thread = after_successful_connection do
|
312
325
|
begin
|
@@ -320,18 +333,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
320
333
|
end
|
321
334
|
end
|
322
335
|
|
323
|
-
|
324
|
-
@dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
|
325
|
-
|
326
|
-
@dlq_codes = DOC_DLQ_CODES.to_set
|
327
|
-
|
328
|
-
if dlq_enabled?
|
329
|
-
check_dlq_custom_codes
|
330
|
-
@dlq_codes.merge(dlq_custom_codes)
|
331
|
-
else
|
332
|
-
raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
|
333
|
-
end
|
336
|
+
end
|
334
337
|
|
338
|
+
def setup_mapper_and_target(data_stream_enabled)
|
335
339
|
if data_stream_enabled
|
336
340
|
@event_mapper = -> (e) { data_stream_event_action_tuple(e) }
|
337
341
|
@event_target = -> (e) { data_stream_name(e) }
|
@@ -340,14 +344,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
340
344
|
@event_mapper = -> (e) { event_action_tuple(e) }
|
341
345
|
@event_target = -> (e) { e.sprintf(@index) }
|
342
346
|
end
|
343
|
-
|
344
|
-
@bulk_request_metrics = metric.namespace(:bulk_requests)
|
345
|
-
@document_level_metrics = metric.namespace(:documents)
|
346
|
-
|
347
|
-
if ecs_compatibility == :v8
|
348
|
-
@logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
|
349
|
-
"Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
|
350
|
-
end
|
351
347
|
end
|
352
348
|
|
353
349
|
# @override post-register when ES connection established
|
@@ -395,7 +391,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
395
391
|
|
396
392
|
event_mapping_errors.each do |event_mapping_error|
|
397
393
|
detailed_message = "#{event_mapping_error.message}; event: `#{event_mapping_error.event.to_hash_with_metadata}`"
|
398
|
-
|
394
|
+
@dlq_writer ? @dlq_writer.write(event_mapping_error.event, detailed_message) : @logger.warn(detailed_message)
|
399
395
|
end
|
400
396
|
@document_level_metrics.increment(:non_retryable_failures, event_mapping_errors.size)
|
401
397
|
end
|
@@ -412,7 +408,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
412
408
|
successful_events << @event_mapper.call(event)
|
413
409
|
rescue EventMappingError => ie
|
414
410
|
event_mapping_errors << FailedEventMapping.new(event, ie.message)
|
415
|
-
|
411
|
+
end
|
416
412
|
end
|
417
413
|
MapEventsResult.new(successful_events, event_mapping_errors)
|
418
414
|
end
|
@@ -425,7 +421,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
425
421
|
def wait_for_successful_connection
|
426
422
|
after_successful_connection_done = @after_successful_connection_done
|
427
423
|
return unless after_successful_connection_done
|
428
|
-
stoppable_sleep 1 until after_successful_connection_done.true?
|
424
|
+
stoppable_sleep 1 until (after_successful_connection_done.true? || pipeline_shutdown_requested?)
|
425
|
+
|
426
|
+
if pipeline_shutdown_requested? && !after_successful_connection_done.true?
|
427
|
+
logger.info "Aborting the batch due to shutdown request while waiting for connections to become live"
|
428
|
+
abort_batch_if_available!
|
429
|
+
end
|
429
430
|
|
430
431
|
status = @after_successful_connection_thread && @after_successful_connection_thread.value
|
431
432
|
if status.is_a?(Exception) # check if thread 'halted' with an error
|
@@ -524,19 +525,22 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
524
525
|
routing_field_name => @routing ? event.sprintf(@routing) : nil
|
525
526
|
}
|
526
527
|
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
params[:pipeline] = value unless value.empty?
|
535
|
-
end
|
528
|
+
target_pipeline = resolve_pipeline(event)
|
529
|
+
# convention: empty string equates to not using a pipeline
|
530
|
+
# this is useful when using a field reference in the pipeline setting, e.g.
|
531
|
+
# elasticsearch {
|
532
|
+
# pipeline => "%{[@metadata][pipeline]}"
|
533
|
+
# }
|
534
|
+
params[:pipeline] = target_pipeline unless (target_pipeline.nil? || target_pipeline.empty?)
|
536
535
|
|
537
536
|
params
|
538
537
|
end
|
539
538
|
|
539
|
+
def resolve_pipeline(event)
|
540
|
+
pipeline_template = @pipeline || event.get("[@metadata][target_ingest_pipeline]")&.to_s
|
541
|
+
pipeline_template && event.sprintf(pipeline_template)
|
542
|
+
end
|
543
|
+
|
540
544
|
@@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
|
541
545
|
|
542
546
|
@@plugins.each do |plugin|
|
@@ -619,6 +623,52 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
619
623
|
end
|
620
624
|
end
|
621
625
|
|
626
|
+
def setup_ssl_params!
|
627
|
+
@ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
|
628
|
+
normalize.with_deprecated_alias(:ssl)
|
629
|
+
end
|
630
|
+
|
631
|
+
@ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
|
632
|
+
normalize.with_deprecated_mapping(:cacert) do |cacert|
|
633
|
+
[cacert]
|
634
|
+
end
|
635
|
+
end
|
636
|
+
|
637
|
+
@ssl_keystore_path = normalize_config(:ssl_keystore_path) do |normalize|
|
638
|
+
normalize.with_deprecated_alias(:keystore)
|
639
|
+
end
|
640
|
+
|
641
|
+
@ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize|
|
642
|
+
normalize.with_deprecated_alias(:keystore_password)
|
643
|
+
end
|
644
|
+
|
645
|
+
@ssl_truststore_path = normalize_config(:ssl_truststore_path) do |normalize|
|
646
|
+
normalize.with_deprecated_alias(:truststore)
|
647
|
+
end
|
648
|
+
|
649
|
+
@ssl_truststore_password = normalize_config(:ssl_truststore_password) do |normalize|
|
650
|
+
normalize.with_deprecated_alias(:truststore_password)
|
651
|
+
end
|
652
|
+
|
653
|
+
@ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize|
|
654
|
+
normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification|
|
655
|
+
if ssl_certificate_verification == true
|
656
|
+
"full"
|
657
|
+
else
|
658
|
+
"none"
|
659
|
+
end
|
660
|
+
end
|
661
|
+
end
|
662
|
+
|
663
|
+
params['ssl_enabled'] = @ssl_enabled unless @ssl_enabled.nil?
|
664
|
+
params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
|
665
|
+
params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil?
|
666
|
+
params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil?
|
667
|
+
params['ssl_truststore_path'] = @ssl_truststore_path unless @ssl_truststore_path.nil?
|
668
|
+
params['ssl_truststore_password'] = @ssl_truststore_password unless @ssl_truststore_password.nil?
|
669
|
+
params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
|
670
|
+
end
|
671
|
+
|
622
672
|
# To be overidden by the -java version
|
623
673
|
VALID_HTTP_ACTIONS = ["index", "delete", "create", "update"]
|
624
674
|
def valid_actions
|
@@ -45,35 +45,79 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
45
45
|
# Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
|
46
46
|
# is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
|
47
47
|
# If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
|
48
|
-
:ssl => { :validate => :boolean },
|
48
|
+
:ssl => { :validate => :boolean, :deprecated => "Set 'ssl_enabled' instead." },
|
49
|
+
|
50
|
+
# Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
|
51
|
+
# is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
|
52
|
+
# If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
|
53
|
+
:ssl_enabled => { :validate => :boolean },
|
49
54
|
|
50
55
|
# Option to validate the server's certificate. Disabling this severely compromises security.
|
51
56
|
# For more information on disabling certificate verification please read
|
52
57
|
# https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
|
53
|
-
:ssl_certificate_verification => { :validate => :boolean, :default => true },
|
58
|
+
:ssl_certificate_verification => { :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead." },
|
59
|
+
|
60
|
+
# Options to verify the server's certificate.
|
61
|
+
# "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
|
62
|
+
# chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
|
63
|
+
# "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
|
64
|
+
:ssl_verification_mode => { :validate => %w[full none], :default => 'full' },
|
54
65
|
|
55
66
|
# The .cer or .pem file to validate the server's certificate
|
56
|
-
:cacert => { :validate => :path },
|
67
|
+
:cacert => { :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." },
|
68
|
+
|
69
|
+
# The .cer or .pem files to validate the server's certificate
|
70
|
+
:ssl_certificate_authorities => { :validate => :path, :list => true },
|
57
71
|
|
58
72
|
# One or more hex-encoded SHA256 fingerprints to trust as Certificate Authorities
|
59
73
|
:ca_trusted_fingerprint => LogStash::PluginMixins::CATrustedFingerprintSupport,
|
60
74
|
|
61
75
|
# The JKS truststore to validate the server's certificate.
|
62
76
|
# Use either `:truststore` or `:cacert`
|
63
|
-
:truststore => { :validate => :path },
|
77
|
+
:truststore => { :validate => :path, :deprecated => "Set 'ssl_truststore_path' instead." },
|
78
|
+
|
79
|
+
# The JKS truststore to validate the server's certificate.
|
80
|
+
# Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
|
81
|
+
:ssl_truststore_path => { :validate => :path },
|
82
|
+
|
83
|
+
# The format of the truststore file. It must be either jks or pkcs12
|
84
|
+
:ssl_truststore_type => { :validate => %w[pkcs12 jks] },
|
85
|
+
|
86
|
+
# Set the truststore password
|
87
|
+
:truststore_password => { :validate => :password, :deprecated => "Use 'ssl_truststore_password' instead." },
|
64
88
|
|
65
89
|
# Set the truststore password
|
66
|
-
:
|
90
|
+
:ssl_truststore_password => { :validate => :password },
|
67
91
|
|
68
92
|
# The keystore used to present a certificate to the server.
|
69
93
|
# It can be either .jks or .p12
|
70
|
-
:keystore => { :validate => :path },
|
94
|
+
:keystore => { :validate => :path, :deprecated => "Set 'ssl_keystore_path' instead." },
|
95
|
+
|
96
|
+
# The keystore used to present a certificate to the server.
|
97
|
+
# It can be either .jks or .p12
|
98
|
+
:ssl_keystore_path => { :validate => :path },
|
99
|
+
|
100
|
+
# The format of the keystore file. It must be either jks or pkcs12
|
101
|
+
:ssl_keystore_type => { :validate => %w[pkcs12 jks] },
|
102
|
+
|
103
|
+
# Set the keystore password
|
104
|
+
:keystore_password => { :validate => :password, :deprecated => "Set 'ssl_keystore_password' instead." },
|
71
105
|
|
72
106
|
# Set the keystore password
|
73
|
-
:
|
107
|
+
:ssl_keystore_password => { :validate => :password },
|
74
108
|
|
75
109
|
:ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true },
|
76
110
|
|
111
|
+
# OpenSSL-style X.509 certificate certificate to authenticate the client
|
112
|
+
:ssl_certificate => { :validate => :path },
|
113
|
+
|
114
|
+
# OpenSSL-style RSA private key to authenticate the client
|
115
|
+
:ssl_key => { :validate => :path },
|
116
|
+
|
117
|
+
# The list of cipher suites to use, listed by priorities.
|
118
|
+
# Supported cipher suites vary depending on which version of Java is used.
|
119
|
+
:ssl_cipher_suites => { :validate => :string, :list => true },
|
120
|
+
|
77
121
|
# This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
|
78
122
|
# Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
|
79
123
|
# this with master nodes, you probably want to disable HTTP on them by setting
|
@@ -169,7 +213,15 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
169
213
|
:retry_initial_interval => { :validate => :number, :default => 2 },
|
170
214
|
|
171
215
|
# Set max interval in seconds between bulk retries.
|
172
|
-
:retry_max_interval => { :validate => :number, :default => 64 }
|
216
|
+
:retry_max_interval => { :validate => :number, :default => 64 },
|
217
|
+
|
218
|
+
# List extra HTTP's error codes that are considered valid to move the events into the dead letter queue.
|
219
|
+
# It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
|
220
|
+
# The option accepts a list of natural numbers corresponding to HTTP errors codes.
|
221
|
+
:dlq_custom_codes => { :validate => :number, :list => true, :default => [] },
|
222
|
+
|
223
|
+
# if enabled, failed index name interpolation events go into dead letter queue.
|
224
|
+
:dlq_on_failed_indexname_interpolation => { :validate => :boolean, :default => true }
|
173
225
|
}.freeze
|
174
226
|
|
175
227
|
def self.included(base)
|
@@ -28,8 +28,7 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
28
28
|
|
29
29
|
setup_hosts
|
30
30
|
|
31
|
-
|
32
|
-
params['ssl'] = effectively_ssl? unless params.include?('ssl')
|
31
|
+
params['ssl_enabled'] = effectively_ssl? unless params.include?('ssl_enabled')
|
33
32
|
|
34
33
|
# inject the TrustStrategy from CATrustedFingerprintSupport
|
35
34
|
if trust_strategy_for_ca_trusted_fingerprint
|
@@ -74,7 +73,7 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
74
73
|
end
|
75
74
|
|
76
75
|
def effectively_ssl?
|
77
|
-
return @
|
76
|
+
return @ssl_enabled unless @ssl_enabled.nil?
|
78
77
|
|
79
78
|
hosts = Array(@hosts)
|
80
79
|
return false if hosts.nil? || hosts.empty?
|
@@ -160,6 +159,8 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
160
159
|
def after_successful_connection(&block)
|
161
160
|
Thread.new do
|
162
161
|
sleep_interval = @retry_initial_interval
|
162
|
+
# in case of a pipeline's shutdown_requested?, the method #close shutdown also this thread
|
163
|
+
# so no need to explicitly handle it here and return an AbortedBatchException.
|
163
164
|
until successful_connection? || @stopping.true?
|
164
165
|
@logger.debug("Waiting for connectivity to Elasticsearch cluster, retrying in #{sleep_interval}s")
|
165
166
|
sleep_interval = sleep_for_interval(sleep_interval)
|
@@ -192,8 +193,14 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
192
193
|
@logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request", count: submit_actions.size)
|
193
194
|
end
|
194
195
|
rescue => e
|
195
|
-
|
196
|
-
|
196
|
+
if abort_batch_present? && e.instance_of?(org.logstash.execution.AbortedBatchException)
|
197
|
+
# if Logstash support abort of a batch and the batch is aborting,
|
198
|
+
# bubble up the exception so that the pipeline can handle it
|
199
|
+
raise e
|
200
|
+
else
|
201
|
+
@logger.error("Encountered an unexpected error submitting a bulk request, will retry",
|
202
|
+
message: e.message, exception: e.class, backtrace: e.backtrace)
|
203
|
+
end
|
197
204
|
end
|
198
205
|
|
199
206
|
# Everything was a success!
|
@@ -220,22 +227,16 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
220
227
|
end
|
221
228
|
|
222
229
|
def handle_dlq_response(message, action, status, response)
|
223
|
-
|
224
|
-
|
225
|
-
# TODO: Change this to send a map with { :status => status, :action => action } in the future
|
226
|
-
detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
|
227
|
-
|
228
|
-
log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
|
230
|
+
event, action_params = action.event, [action[0], action[1], action[2]]
|
229
231
|
|
230
|
-
handle_dlq_status(action.event, log_level, detailed_message)
|
231
|
-
end
|
232
|
-
|
233
|
-
def handle_dlq_status(event, log_level, message)
|
234
|
-
# To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
|
235
232
|
if @dlq_writer
|
236
|
-
|
233
|
+
# TODO: Change this to send a map with { :status => status, :action => action } in the future
|
234
|
+
detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
|
235
|
+
@dlq_writer.write(event, "#{detailed_message}")
|
237
236
|
else
|
238
|
-
|
237
|
+
log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
|
238
|
+
|
239
|
+
@logger.public_send(log_level, message, status: status, action: action_params, response: response)
|
239
240
|
end
|
240
241
|
end
|
241
242
|
|
@@ -332,6 +333,11 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
332
333
|
|
333
334
|
sleep_interval = sleep_for_interval(sleep_interval)
|
334
335
|
@bulk_request_metrics.increment(:failures)
|
336
|
+
if pipeline_shutdown_requested?
|
337
|
+
# when any connection is available and a shutdown is requested
|
338
|
+
# the batch can be aborted, eventually for future retry.
|
339
|
+
abort_batch_if_available!
|
340
|
+
end
|
335
341
|
retry unless @stopping.true?
|
336
342
|
rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
|
337
343
|
@bulk_request_metrics.increment(:failures)
|
@@ -350,6 +356,11 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
350
356
|
end
|
351
357
|
|
352
358
|
sleep_interval = sleep_for_interval(sleep_interval)
|
359
|
+
if pipeline_shutdown_requested?
|
360
|
+
# In case ES side changes access credentials and a pipeline reload is triggered
|
361
|
+
# this error becomes a retry on restart
|
362
|
+
abort_batch_if_available!
|
363
|
+
end
|
353
364
|
retry
|
354
365
|
rescue => e # Stuff that should never happen - print out full connection issues
|
355
366
|
@logger.error(
|
@@ -364,6 +375,19 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
364
375
|
end
|
365
376
|
end
|
366
377
|
|
378
|
+
def pipeline_shutdown_requested?
|
379
|
+
return super if defined?(super) # since LS 8.1.0
|
380
|
+
execution_context&.pipeline&.shutdown_requested?
|
381
|
+
end
|
382
|
+
|
383
|
+
def abort_batch_if_available!
|
384
|
+
raise org.logstash.execution.AbortedBatchException.new if abort_batch_present?
|
385
|
+
end
|
386
|
+
|
387
|
+
def abort_batch_present?
|
388
|
+
::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.8.0')
|
389
|
+
end
|
390
|
+
|
367
391
|
def dlq_enabled?
|
368
392
|
# TODO there should be a better way to query if DLQ is enabled
|
369
393
|
# See more in: https://github.com/elastic/logstash/issues/8064
|
@@ -1,12 +1,12 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-elasticsearch'
|
3
|
-
s.version = '11.
|
3
|
+
s.version = '11.15.9'
|
4
4
|
s.licenses = ['apache-2.0']
|
5
5
|
s.summary = "Stores logs in Elasticsearch"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
7
|
s.authors = ["Elastic"]
|
8
8
|
s.email = 'info@elastic.co'
|
9
|
-
s.homepage = "
|
9
|
+
s.homepage = "https://www.elastic.co/guide/en/logstash/current/index.html"
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
s.platform = RUBY_PLATFORM
|
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
|
|
26
26
|
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
|
27
27
|
s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
|
28
28
|
s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~>1.0'
|
29
|
+
s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
|
29
30
|
|
30
31
|
s.add_development_dependency 'logstash-codec-plain'
|
31
32
|
s.add_development_dependency 'logstash-devutils'
|
data/spec/es_spec_helper.rb
CHANGED
@@ -67,19 +67,24 @@ module ESHelper
|
|
67
67
|
end
|
68
68
|
|
69
69
|
RSpec::Matchers.define :have_hits do |expected|
|
70
|
+
hits_count_path = ESHelper.es_version_satisfies?(">=7") ? %w(hits total value) : %w(hits total)
|
71
|
+
|
70
72
|
match do |actual|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
73
|
+
@actual_hits_count = actual&.dig(*hits_count_path)
|
74
|
+
values_match? expected, @actual_hits_count
|
75
|
+
end
|
76
|
+
failure_message do |actual|
|
77
|
+
"expected that #{actual} with #{@actual_hits_count || "UNKNOWN" } hits would have #{expected} hits"
|
76
78
|
end
|
77
79
|
end
|
78
80
|
|
79
81
|
RSpec::Matchers.define :have_index_pattern do |expected|
|
80
82
|
match do |actual|
|
81
|
-
|
82
|
-
|
83
|
+
@actual_index_pattterns = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
|
84
|
+
@actual_index_pattterns.any? { |v| values_match? expected, v }
|
85
|
+
end
|
86
|
+
failure_message do |actual|
|
87
|
+
"expected that #{actual} with index patterns #{@actual_index_pattterns} would have included `#{expected}`"
|
83
88
|
end
|
84
89
|
end
|
85
90
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
2
|
-
|
2
|
+
MIIFDDCCAvQCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
|
3
3
|
BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
|
4
|
-
|
4
|
+
DAZSb290Q0EwHhcNMjMwNTMwMTUxMDM4WhcNMjQwNTI5MTUxMDM4WjBMMQswCQYD
|
5
5
|
VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
|
6
6
|
TXlMYWIxDzANBgNVBAMMBlJvb3RDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
|
7
7
|
AgoCggIBAMtTMqAWuH17b9XqPa5L3HNqgnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh
|
@@ -15,15 +15,15 @@ dY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aLojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv
|
|
15
15
|
5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nxUjCbgn0RABojRoGdhhY3QDipgwmSgFZx
|
16
16
|
r064RFr1bt/Ml3MJmPf535mSwPdk/j/zw4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMP
|
17
17
|
LEdi73Qjzmr3ENIrir4O86wNz81YRfYkg9ZX8yKJK9LBAUrYCjJ3AgMBAAEwDQYJ
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
18
|
+
KoZIhvcNAQELBQADggIBACE5DihUXpysZv0d25Fw9V2LRI0iJXOoVOu+RrnkL2HD
|
19
|
+
LGBEbw0KOkS3mpgKpir1pD4TINAvPs5ZkAAREny8bAXrhdUY6Gd+Fpq5bwPnZHr3
|
20
|
+
UvazLCNY5YUQpg1TjgbQ9LyBwf5jz5ZHR6Kilw87kaAdqzgqRnMXOuuSZeT70vH+
|
21
|
+
M+Ra99lLpyT+A2Isp7/vzg3HhSAi/UsZCPzGLQwEeZBmlaKAtsjF0B1L8cvd5xWh
|
22
|
+
rZ3PJWfvn2Aaiz4QEVq+jiZW8Y6bqHDb/lZQPs9Z5dLOww56VDJcBU87mayAYnRH
|
23
|
+
edsshuCVqwTZU3Y3+z/g/G+IQWByYM9sr3zcgpFdI74Ly20ClbegqaXXL1wfhA76
|
24
|
+
zT4cLH616Ukdqi8bCPPgy5KnYQWjks8cvabjPT/HeHzhJ/2vkfb1vWGHBCU9fg1n
|
25
|
+
mfVWvRJlf2McwW2vogE3eHFnEJWOha85Kif/SteVH0cHHHIUacJhtD6m0wIDW7vU
|
26
|
+
1xjDkipzKGnOsxGjLxAvw/eyHHWx8XT+z7oPzQX2UBStsIB4WGYmqqW3tV19E8Li
|
27
|
+
bGk5klu+lXK0UomAm2MD3MRR10UCkVFXM4/cUfiMrAgG232yDwRLiGp1EmY3uHyD
|
28
|
+
8/5mRJzBtLsGQKbfBPPNExiFqDzXr2ZwE7tyfsB8auSV3mkVjYjYYFnDfE835U+y
|
29
29
|
-----END CERTIFICATE-----
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
86a8abdffc0dd114d50cb20c7cc635bdb4bdcb16370fdee5aa5c05b4861faacd
|
@@ -1,7 +1,7 @@
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
2
|
-
|
2
|
+
MIIFEzCCAvsCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
|
3
3
|
BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
|
4
|
-
|
4
|
+
DAZSb290Q0EwHhcNMjMwNTMwMTUxMDM4WhcNMjQwNTI5MTUxMDM4WjBTMQswCQYD
|
5
5
|
VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
|
6
6
|
TXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEBAQUA
|
7
7
|
A4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFcV0r2
|
@@ -15,16 +15,16 @@ QcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcnM6EO
|
|
15
15
|
fy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYicNUe
|
16
16
|
cqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkSjsn3
|
17
17
|
cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV1wID
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
18
|
+
AQABMA0GCSqGSIb3DQEBCwUAA4ICAQC4pZQWJoyNANFscsmm6I/u5LgerRCnCS+Y
|
19
|
+
V0tLWribc/iHQNzmUygvwT8+tllp+OzWmp/7oDuFrD/HFf8Xmaj14LTS2QVhZPao
|
20
|
+
NMxkcq7W1WRnTdT+4Do4QE3l5hClgfOvcKzzrhCIPQ//aYgUo5JmXUsYoIWSsJ6v
|
21
|
+
Z77j70xsz95k+C2qfiIKVx6a0U0lXjkp1fwrpGsG0dUL8GAibcX+BYD91uTMTAmg
|
22
|
+
SB4O2xMR6kKCCH3SDuRnkI+iC4MZL64JTrGzn+juMeK0fqZWrdyoVKRlFI92Ao3J
|
23
|
+
oFNF9KTCWQPFAaiy1zl3vDAfIfyJBzQsP6D2BaxVlSQ5jbCxRxOapkhwNUUQ2/vg
|
24
|
+
anggOIOnm6biKhain4U118jviSEGonTvzcIpjExLVXXTF216Ahd7vQWDWqZkYNi3
|
25
|
+
ji1jZSemBWyv/bDxo/STLL/QiX73WuxelerZeMxS03U6KYyhtTLxDvpeOCstbe1u
|
26
|
+
wI9LfpdtAbXP+d/djvYJSHb6WoS4Oya2mnyddqDd6Mfn7lTudhqtw5CLV+sElnFf
|
27
|
+
XDBjDp6tWUin6s/IWPVnvMceZW/sW7oO3sy9NKQx1acHh85WIMrZ+9iVvwcvQMRF
|
28
|
+
p/bTR2VL3jD5fioNyfBmmQ17wIOIm3QvY+nJmzOWalzYYzottPTiabzKYap7Ulcx
|
29
|
+
HDB+ZvuJBA==
|
30
30
|
-----END CERTIFICATE-----
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
b6ee8b4a213328074df1da1f3b8fdde95c5e97e9b1d7da58dad0537a04939b97
|
Binary file
|