logstash-output-elasticsearch 11.12.4-java → 11.15.9-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +58 -0
- data/docs/index.asciidoc +214 -66
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +14 -4
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +46 -19
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +26 -3
- data/lib/logstash/outputs/elasticsearch.rb +89 -39
- data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb +60 -8
- data/lib/logstash/plugin_mixins/elasticsearch/common.rb +42 -18
- data/logstash-output-elasticsearch.gemspec +3 -2
- data/spec/es_spec_helper.rb +12 -7
- data/spec/fixtures/test_certs/ca.crt +13 -13
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -1
- data/spec/fixtures/test_certs/test.crt +14 -14
- data/spec/fixtures/test_certs/test.der.sha256 +1 -1
- data/spec/fixtures/test_certs/test.p12 +0 -0
- data/spec/integration/outputs/index_spec.rb +16 -16
- data/spec/integration/outputs/templates_spec.rb +11 -9
- data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb +4 -2
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +9 -2
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +72 -20
- data/spec/unit/outputs/elasticsearch_spec.rb +358 -28
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +166 -50
- metadata +18 -4
@@ -96,10 +96,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
96
96
|
require "logstash/outputs/elasticsearch/data_stream_support"
|
97
97
|
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
98
98
|
require 'logstash/plugin_mixins/deprecation_logger_support'
|
99
|
+
require 'logstash/plugin_mixins/normalize_config_support'
|
99
100
|
|
100
101
|
# Protocol agnostic methods
|
101
102
|
include(LogStash::PluginMixins::ElasticSearch::Common)
|
102
103
|
|
104
|
+
# Config normalization helpers
|
105
|
+
include(LogStash::PluginMixins::NormalizeConfigSupport)
|
106
|
+
|
103
107
|
# Methods for ILM support
|
104
108
|
include(LogStash::Outputs::ElasticSearch::Ilm)
|
105
109
|
|
@@ -263,14 +267,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
263
267
|
# ILM policy to use, if undefined the default policy will be used.
|
264
268
|
config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
|
265
269
|
|
266
|
-
# List extra HTTP's error codes that are considered valid to move the events into the dead letter queue.
|
267
|
-
# It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
|
268
|
-
# The option accepts a list of natural numbers corresponding to HTTP errors codes.
|
269
|
-
config :dlq_custom_codes, :validate => :number, :list => true, :default => []
|
270
|
-
|
271
|
-
# if enabled, failed index name interpolation events go into dead letter queue.
|
272
|
-
config :dlq_on_failed_indexname_interpolation, :validate => :boolean, :default => true
|
273
|
-
|
274
270
|
attr_reader :client
|
275
271
|
attr_reader :default_index
|
276
272
|
attr_reader :default_ilm_rollover_alias
|
@@ -279,6 +275,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
279
275
|
def initialize(*params)
|
280
276
|
super
|
281
277
|
setup_ecs_compatibility_related_defaults
|
278
|
+
setup_ssl_params!
|
282
279
|
end
|
283
280
|
|
284
281
|
def register
|
@@ -307,6 +304,22 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
307
304
|
data_stream_enabled = data_stream_config?
|
308
305
|
|
309
306
|
setup_template_manager_defaults(data_stream_enabled)
|
307
|
+
# To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
|
308
|
+
@dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
|
309
|
+
|
310
|
+
@dlq_codes = DOC_DLQ_CODES.to_set
|
311
|
+
|
312
|
+
if dlq_enabled?
|
313
|
+
check_dlq_custom_codes
|
314
|
+
@dlq_codes.merge(dlq_custom_codes)
|
315
|
+
else
|
316
|
+
raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
|
317
|
+
end
|
318
|
+
|
319
|
+
setup_mapper_and_target(data_stream_enabled)
|
320
|
+
|
321
|
+
@bulk_request_metrics = metric.namespace(:bulk_requests)
|
322
|
+
@document_level_metrics = metric.namespace(:documents)
|
310
323
|
|
311
324
|
@after_successful_connection_thread = after_successful_connection do
|
312
325
|
begin
|
@@ -320,18 +333,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
320
333
|
end
|
321
334
|
end
|
322
335
|
|
323
|
-
|
324
|
-
@dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
|
325
|
-
|
326
|
-
@dlq_codes = DOC_DLQ_CODES.to_set
|
327
|
-
|
328
|
-
if dlq_enabled?
|
329
|
-
check_dlq_custom_codes
|
330
|
-
@dlq_codes.merge(dlq_custom_codes)
|
331
|
-
else
|
332
|
-
raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
|
333
|
-
end
|
336
|
+
end
|
334
337
|
|
338
|
+
def setup_mapper_and_target(data_stream_enabled)
|
335
339
|
if data_stream_enabled
|
336
340
|
@event_mapper = -> (e) { data_stream_event_action_tuple(e) }
|
337
341
|
@event_target = -> (e) { data_stream_name(e) }
|
@@ -340,14 +344,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
340
344
|
@event_mapper = -> (e) { event_action_tuple(e) }
|
341
345
|
@event_target = -> (e) { e.sprintf(@index) }
|
342
346
|
end
|
343
|
-
|
344
|
-
@bulk_request_metrics = metric.namespace(:bulk_requests)
|
345
|
-
@document_level_metrics = metric.namespace(:documents)
|
346
|
-
|
347
|
-
if ecs_compatibility == :v8
|
348
|
-
@logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
|
349
|
-
"Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
|
350
|
-
end
|
351
347
|
end
|
352
348
|
|
353
349
|
# @override post-register when ES connection established
|
@@ -395,7 +391,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
395
391
|
|
396
392
|
event_mapping_errors.each do |event_mapping_error|
|
397
393
|
detailed_message = "#{event_mapping_error.message}; event: `#{event_mapping_error.event.to_hash_with_metadata}`"
|
398
|
-
|
394
|
+
@dlq_writer ? @dlq_writer.write(event_mapping_error.event, detailed_message) : @logger.warn(detailed_message)
|
399
395
|
end
|
400
396
|
@document_level_metrics.increment(:non_retryable_failures, event_mapping_errors.size)
|
401
397
|
end
|
@@ -412,7 +408,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
412
408
|
successful_events << @event_mapper.call(event)
|
413
409
|
rescue EventMappingError => ie
|
414
410
|
event_mapping_errors << FailedEventMapping.new(event, ie.message)
|
415
|
-
|
411
|
+
end
|
416
412
|
end
|
417
413
|
MapEventsResult.new(successful_events, event_mapping_errors)
|
418
414
|
end
|
@@ -425,7 +421,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
425
421
|
def wait_for_successful_connection
|
426
422
|
after_successful_connection_done = @after_successful_connection_done
|
427
423
|
return unless after_successful_connection_done
|
428
|
-
stoppable_sleep 1 until after_successful_connection_done.true?
|
424
|
+
stoppable_sleep 1 until (after_successful_connection_done.true? || pipeline_shutdown_requested?)
|
425
|
+
|
426
|
+
if pipeline_shutdown_requested? && !after_successful_connection_done.true?
|
427
|
+
logger.info "Aborting the batch due to shutdown request while waiting for connections to become live"
|
428
|
+
abort_batch_if_available!
|
429
|
+
end
|
429
430
|
|
430
431
|
status = @after_successful_connection_thread && @after_successful_connection_thread.value
|
431
432
|
if status.is_a?(Exception) # check if thread 'halted' with an error
|
@@ -524,19 +525,22 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
524
525
|
routing_field_name => @routing ? event.sprintf(@routing) : nil
|
525
526
|
}
|
526
527
|
|
527
|
-
|
528
|
-
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
params[:pipeline] = value unless value.empty?
|
535
|
-
end
|
528
|
+
target_pipeline = resolve_pipeline(event)
|
529
|
+
# convention: empty string equates to not using a pipeline
|
530
|
+
# this is useful when using a field reference in the pipeline setting, e.g.
|
531
|
+
# elasticsearch {
|
532
|
+
# pipeline => "%{[@metadata][pipeline]}"
|
533
|
+
# }
|
534
|
+
params[:pipeline] = target_pipeline unless (target_pipeline.nil? || target_pipeline.empty?)
|
536
535
|
|
537
536
|
params
|
538
537
|
end
|
539
538
|
|
539
|
+
def resolve_pipeline(event)
|
540
|
+
pipeline_template = @pipeline || event.get("[@metadata][target_ingest_pipeline]")&.to_s
|
541
|
+
pipeline_template && event.sprintf(pipeline_template)
|
542
|
+
end
|
543
|
+
|
540
544
|
@@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
|
541
545
|
|
542
546
|
@@plugins.each do |plugin|
|
@@ -619,6 +623,52 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
619
623
|
end
|
620
624
|
end
|
621
625
|
|
626
|
+
def setup_ssl_params!
|
627
|
+
@ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
|
628
|
+
normalize.with_deprecated_alias(:ssl)
|
629
|
+
end
|
630
|
+
|
631
|
+
@ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
|
632
|
+
normalize.with_deprecated_mapping(:cacert) do |cacert|
|
633
|
+
[cacert]
|
634
|
+
end
|
635
|
+
end
|
636
|
+
|
637
|
+
@ssl_keystore_path = normalize_config(:ssl_keystore_path) do |normalize|
|
638
|
+
normalize.with_deprecated_alias(:keystore)
|
639
|
+
end
|
640
|
+
|
641
|
+
@ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize|
|
642
|
+
normalize.with_deprecated_alias(:keystore_password)
|
643
|
+
end
|
644
|
+
|
645
|
+
@ssl_truststore_path = normalize_config(:ssl_truststore_path) do |normalize|
|
646
|
+
normalize.with_deprecated_alias(:truststore)
|
647
|
+
end
|
648
|
+
|
649
|
+
@ssl_truststore_password = normalize_config(:ssl_truststore_password) do |normalize|
|
650
|
+
normalize.with_deprecated_alias(:truststore_password)
|
651
|
+
end
|
652
|
+
|
653
|
+
@ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize|
|
654
|
+
normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification|
|
655
|
+
if ssl_certificate_verification == true
|
656
|
+
"full"
|
657
|
+
else
|
658
|
+
"none"
|
659
|
+
end
|
660
|
+
end
|
661
|
+
end
|
662
|
+
|
663
|
+
params['ssl_enabled'] = @ssl_enabled unless @ssl_enabled.nil?
|
664
|
+
params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
|
665
|
+
params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil?
|
666
|
+
params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil?
|
667
|
+
params['ssl_truststore_path'] = @ssl_truststore_path unless @ssl_truststore_path.nil?
|
668
|
+
params['ssl_truststore_password'] = @ssl_truststore_password unless @ssl_truststore_password.nil?
|
669
|
+
params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
|
670
|
+
end
|
671
|
+
|
622
672
|
# To be overidden by the -java version
|
623
673
|
VALID_HTTP_ACTIONS = ["index", "delete", "create", "update"]
|
624
674
|
def valid_actions
|
@@ -45,35 +45,79 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
45
45
|
# Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
|
46
46
|
# is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
|
47
47
|
# If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
|
48
|
-
:ssl => { :validate => :boolean },
|
48
|
+
:ssl => { :validate => :boolean, :deprecated => "Set 'ssl_enabled' instead." },
|
49
|
+
|
50
|
+
# Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
|
51
|
+
# is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
|
52
|
+
# If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
|
53
|
+
:ssl_enabled => { :validate => :boolean },
|
49
54
|
|
50
55
|
# Option to validate the server's certificate. Disabling this severely compromises security.
|
51
56
|
# For more information on disabling certificate verification please read
|
52
57
|
# https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
|
53
|
-
:ssl_certificate_verification => { :validate => :boolean, :default => true },
|
58
|
+
:ssl_certificate_verification => { :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead." },
|
59
|
+
|
60
|
+
# Options to verify the server's certificate.
|
61
|
+
# "full": validates that the provided certificate has an issue date that’s within the not_before and not_after dates;
|
62
|
+
# chains to a trusted Certificate Authority (CA); has a hostname or IP address that matches the names within the certificate.
|
63
|
+
# "none": performs no certificate validation. Disabling this severely compromises security (https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf)
|
64
|
+
:ssl_verification_mode => { :validate => %w[full none], :default => 'full' },
|
54
65
|
|
55
66
|
# The .cer or .pem file to validate the server's certificate
|
56
|
-
:cacert => { :validate => :path },
|
67
|
+
:cacert => { :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead." },
|
68
|
+
|
69
|
+
# The .cer or .pem files to validate the server's certificate
|
70
|
+
:ssl_certificate_authorities => { :validate => :path, :list => true },
|
57
71
|
|
58
72
|
# One or more hex-encoded SHA256 fingerprints to trust as Certificate Authorities
|
59
73
|
:ca_trusted_fingerprint => LogStash::PluginMixins::CATrustedFingerprintSupport,
|
60
74
|
|
61
75
|
# The JKS truststore to validate the server's certificate.
|
62
76
|
# Use either `:truststore` or `:cacert`
|
63
|
-
:truststore => { :validate => :path },
|
77
|
+
:truststore => { :validate => :path, :deprecated => "Set 'ssl_truststore_path' instead." },
|
78
|
+
|
79
|
+
# The JKS truststore to validate the server's certificate.
|
80
|
+
# Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
|
81
|
+
:ssl_truststore_path => { :validate => :path },
|
82
|
+
|
83
|
+
# The format of the truststore file. It must be either jks or pkcs12
|
84
|
+
:ssl_truststore_type => { :validate => %w[pkcs12 jks] },
|
85
|
+
|
86
|
+
# Set the truststore password
|
87
|
+
:truststore_password => { :validate => :password, :deprecated => "Use 'ssl_truststore_password' instead." },
|
64
88
|
|
65
89
|
# Set the truststore password
|
66
|
-
:
|
90
|
+
:ssl_truststore_password => { :validate => :password },
|
67
91
|
|
68
92
|
# The keystore used to present a certificate to the server.
|
69
93
|
# It can be either .jks or .p12
|
70
|
-
:keystore => { :validate => :path },
|
94
|
+
:keystore => { :validate => :path, :deprecated => "Set 'ssl_keystore_path' instead." },
|
95
|
+
|
96
|
+
# The keystore used to present a certificate to the server.
|
97
|
+
# It can be either .jks or .p12
|
98
|
+
:ssl_keystore_path => { :validate => :path },
|
99
|
+
|
100
|
+
# The format of the keystore file. It must be either jks or pkcs12
|
101
|
+
:ssl_keystore_type => { :validate => %w[pkcs12 jks] },
|
102
|
+
|
103
|
+
# Set the keystore password
|
104
|
+
:keystore_password => { :validate => :password, :deprecated => "Set 'ssl_keystore_password' instead." },
|
71
105
|
|
72
106
|
# Set the keystore password
|
73
|
-
:
|
107
|
+
:ssl_keystore_password => { :validate => :password },
|
74
108
|
|
75
109
|
:ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true },
|
76
110
|
|
111
|
+
# OpenSSL-style X.509 certificate certificate to authenticate the client
|
112
|
+
:ssl_certificate => { :validate => :path },
|
113
|
+
|
114
|
+
# OpenSSL-style RSA private key to authenticate the client
|
115
|
+
:ssl_key => { :validate => :path },
|
116
|
+
|
117
|
+
# The list of cipher suites to use, listed by priorities.
|
118
|
+
# Supported cipher suites vary depending on which version of Java is used.
|
119
|
+
:ssl_cipher_suites => { :validate => :string, :list => true },
|
120
|
+
|
77
121
|
# This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
|
78
122
|
# Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
|
79
123
|
# this with master nodes, you probably want to disable HTTP on them by setting
|
@@ -169,7 +213,15 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
169
213
|
:retry_initial_interval => { :validate => :number, :default => 2 },
|
170
214
|
|
171
215
|
# Set max interval in seconds between bulk retries.
|
172
|
-
:retry_max_interval => { :validate => :number, :default => 64 }
|
216
|
+
:retry_max_interval => { :validate => :number, :default => 64 },
|
217
|
+
|
218
|
+
# List extra HTTP's error codes that are considered valid to move the events into the dead letter queue.
|
219
|
+
# It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
|
220
|
+
# The option accepts a list of natural numbers corresponding to HTTP errors codes.
|
221
|
+
:dlq_custom_codes => { :validate => :number, :list => true, :default => [] },
|
222
|
+
|
223
|
+
# if enabled, failed index name interpolation events go into dead letter queue.
|
224
|
+
:dlq_on_failed_indexname_interpolation => { :validate => :boolean, :default => true }
|
173
225
|
}.freeze
|
174
226
|
|
175
227
|
def self.included(base)
|
@@ -28,8 +28,7 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
28
28
|
|
29
29
|
setup_hosts
|
30
30
|
|
31
|
-
|
32
|
-
params['ssl'] = effectively_ssl? unless params.include?('ssl')
|
31
|
+
params['ssl_enabled'] = effectively_ssl? unless params.include?('ssl_enabled')
|
33
32
|
|
34
33
|
# inject the TrustStrategy from CATrustedFingerprintSupport
|
35
34
|
if trust_strategy_for_ca_trusted_fingerprint
|
@@ -74,7 +73,7 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
74
73
|
end
|
75
74
|
|
76
75
|
def effectively_ssl?
|
77
|
-
return @
|
76
|
+
return @ssl_enabled unless @ssl_enabled.nil?
|
78
77
|
|
79
78
|
hosts = Array(@hosts)
|
80
79
|
return false if hosts.nil? || hosts.empty?
|
@@ -160,6 +159,8 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
160
159
|
def after_successful_connection(&block)
|
161
160
|
Thread.new do
|
162
161
|
sleep_interval = @retry_initial_interval
|
162
|
+
# in case of a pipeline's shutdown_requested?, the method #close shutdown also this thread
|
163
|
+
# so no need to explicitly handle it here and return an AbortedBatchException.
|
163
164
|
until successful_connection? || @stopping.true?
|
164
165
|
@logger.debug("Waiting for connectivity to Elasticsearch cluster, retrying in #{sleep_interval}s")
|
165
166
|
sleep_interval = sleep_for_interval(sleep_interval)
|
@@ -192,8 +193,14 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
192
193
|
@logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request", count: submit_actions.size)
|
193
194
|
end
|
194
195
|
rescue => e
|
195
|
-
|
196
|
-
|
196
|
+
if abort_batch_present? && e.instance_of?(org.logstash.execution.AbortedBatchException)
|
197
|
+
# if Logstash support abort of a batch and the batch is aborting,
|
198
|
+
# bubble up the exception so that the pipeline can handle it
|
199
|
+
raise e
|
200
|
+
else
|
201
|
+
@logger.error("Encountered an unexpected error submitting a bulk request, will retry",
|
202
|
+
message: e.message, exception: e.class, backtrace: e.backtrace)
|
203
|
+
end
|
197
204
|
end
|
198
205
|
|
199
206
|
# Everything was a success!
|
@@ -220,22 +227,16 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
220
227
|
end
|
221
228
|
|
222
229
|
def handle_dlq_response(message, action, status, response)
|
223
|
-
|
224
|
-
|
225
|
-
# TODO: Change this to send a map with { :status => status, :action => action } in the future
|
226
|
-
detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
|
227
|
-
|
228
|
-
log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
|
230
|
+
event, action_params = action.event, [action[0], action[1], action[2]]
|
229
231
|
|
230
|
-
handle_dlq_status(action.event, log_level, detailed_message)
|
231
|
-
end
|
232
|
-
|
233
|
-
def handle_dlq_status(event, log_level, message)
|
234
|
-
# To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
|
235
232
|
if @dlq_writer
|
236
|
-
|
233
|
+
# TODO: Change this to send a map with { :status => status, :action => action } in the future
|
234
|
+
detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
|
235
|
+
@dlq_writer.write(event, "#{detailed_message}")
|
237
236
|
else
|
238
|
-
|
237
|
+
log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
|
238
|
+
|
239
|
+
@logger.public_send(log_level, message, status: status, action: action_params, response: response)
|
239
240
|
end
|
240
241
|
end
|
241
242
|
|
@@ -332,6 +333,11 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
332
333
|
|
333
334
|
sleep_interval = sleep_for_interval(sleep_interval)
|
334
335
|
@bulk_request_metrics.increment(:failures)
|
336
|
+
if pipeline_shutdown_requested?
|
337
|
+
# when any connection is available and a shutdown is requested
|
338
|
+
# the batch can be aborted, eventually for future retry.
|
339
|
+
abort_batch_if_available!
|
340
|
+
end
|
335
341
|
retry unless @stopping.true?
|
336
342
|
rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
|
337
343
|
@bulk_request_metrics.increment(:failures)
|
@@ -350,6 +356,11 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
350
356
|
end
|
351
357
|
|
352
358
|
sleep_interval = sleep_for_interval(sleep_interval)
|
359
|
+
if pipeline_shutdown_requested?
|
360
|
+
# In case ES side changes access credentials and a pipeline reload is triggered
|
361
|
+
# this error becomes a retry on restart
|
362
|
+
abort_batch_if_available!
|
363
|
+
end
|
353
364
|
retry
|
354
365
|
rescue => e # Stuff that should never happen - print out full connection issues
|
355
366
|
@logger.error(
|
@@ -364,6 +375,19 @@ module LogStash; module PluginMixins; module ElasticSearch
|
|
364
375
|
end
|
365
376
|
end
|
366
377
|
|
378
|
+
def pipeline_shutdown_requested?
|
379
|
+
return super if defined?(super) # since LS 8.1.0
|
380
|
+
execution_context&.pipeline&.shutdown_requested?
|
381
|
+
end
|
382
|
+
|
383
|
+
def abort_batch_if_available!
|
384
|
+
raise org.logstash.execution.AbortedBatchException.new if abort_batch_present?
|
385
|
+
end
|
386
|
+
|
387
|
+
def abort_batch_present?
|
388
|
+
::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.8.0')
|
389
|
+
end
|
390
|
+
|
367
391
|
def dlq_enabled?
|
368
392
|
# TODO there should be a better way to query if DLQ is enabled
|
369
393
|
# See more in: https://github.com/elastic/logstash/issues/8064
|
@@ -1,12 +1,12 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-elasticsearch'
|
3
|
-
s.version = '11.
|
3
|
+
s.version = '11.15.9'
|
4
4
|
s.licenses = ['apache-2.0']
|
5
5
|
s.summary = "Stores logs in Elasticsearch"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
7
|
s.authors = ["Elastic"]
|
8
8
|
s.email = 'info@elastic.co'
|
9
|
-
s.homepage = "
|
9
|
+
s.homepage = "https://www.elastic.co/guide/en/logstash/current/index.html"
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
s.platform = RUBY_PLATFORM
|
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
|
|
26
26
|
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
|
27
27
|
s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
|
28
28
|
s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~>1.0'
|
29
|
+
s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
|
29
30
|
|
30
31
|
s.add_development_dependency 'logstash-codec-plain'
|
31
32
|
s.add_development_dependency 'logstash-devutils'
|
data/spec/es_spec_helper.rb
CHANGED
@@ -67,19 +67,24 @@ module ESHelper
|
|
67
67
|
end
|
68
68
|
|
69
69
|
RSpec::Matchers.define :have_hits do |expected|
|
70
|
+
hits_count_path = ESHelper.es_version_satisfies?(">=7") ? %w(hits total value) : %w(hits total)
|
71
|
+
|
70
72
|
match do |actual|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
73
|
+
@actual_hits_count = actual&.dig(*hits_count_path)
|
74
|
+
values_match? expected, @actual_hits_count
|
75
|
+
end
|
76
|
+
failure_message do |actual|
|
77
|
+
"expected that #{actual} with #{@actual_hits_count || "UNKNOWN" } hits would have #{expected} hits"
|
76
78
|
end
|
77
79
|
end
|
78
80
|
|
79
81
|
RSpec::Matchers.define :have_index_pattern do |expected|
|
80
82
|
match do |actual|
|
81
|
-
|
82
|
-
|
83
|
+
@actual_index_pattterns = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
|
84
|
+
@actual_index_pattterns.any? { |v| values_match? expected, v }
|
85
|
+
end
|
86
|
+
failure_message do |actual|
|
87
|
+
"expected that #{actual} with index patterns #{@actual_index_pattterns} would have included `#{expected}`"
|
83
88
|
end
|
84
89
|
end
|
85
90
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
2
|
-
|
2
|
+
MIIFDDCCAvQCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
|
3
3
|
BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
|
4
|
-
|
4
|
+
DAZSb290Q0EwHhcNMjMwNTMwMTUxMDM4WhcNMjQwNTI5MTUxMDM4WjBMMQswCQYD
|
5
5
|
VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
|
6
6
|
TXlMYWIxDzANBgNVBAMMBlJvb3RDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCC
|
7
7
|
AgoCggIBAMtTMqAWuH17b9XqPa5L3HNqgnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh
|
@@ -15,15 +15,15 @@ dY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aLojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv
|
|
15
15
|
5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nxUjCbgn0RABojRoGdhhY3QDipgwmSgFZx
|
16
16
|
r064RFr1bt/Ml3MJmPf535mSwPdk/j/zw4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMP
|
17
17
|
LEdi73Qjzmr3ENIrir4O86wNz81YRfYkg9ZX8yKJK9LBAUrYCjJ3AgMBAAEwDQYJ
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
18
|
+
KoZIhvcNAQELBQADggIBACE5DihUXpysZv0d25Fw9V2LRI0iJXOoVOu+RrnkL2HD
|
19
|
+
LGBEbw0KOkS3mpgKpir1pD4TINAvPs5ZkAAREny8bAXrhdUY6Gd+Fpq5bwPnZHr3
|
20
|
+
UvazLCNY5YUQpg1TjgbQ9LyBwf5jz5ZHR6Kilw87kaAdqzgqRnMXOuuSZeT70vH+
|
21
|
+
M+Ra99lLpyT+A2Isp7/vzg3HhSAi/UsZCPzGLQwEeZBmlaKAtsjF0B1L8cvd5xWh
|
22
|
+
rZ3PJWfvn2Aaiz4QEVq+jiZW8Y6bqHDb/lZQPs9Z5dLOww56VDJcBU87mayAYnRH
|
23
|
+
edsshuCVqwTZU3Y3+z/g/G+IQWByYM9sr3zcgpFdI74Ly20ClbegqaXXL1wfhA76
|
24
|
+
zT4cLH616Ukdqi8bCPPgy5KnYQWjks8cvabjPT/HeHzhJ/2vkfb1vWGHBCU9fg1n
|
25
|
+
mfVWvRJlf2McwW2vogE3eHFnEJWOha85Kif/SteVH0cHHHIUacJhtD6m0wIDW7vU
|
26
|
+
1xjDkipzKGnOsxGjLxAvw/eyHHWx8XT+z7oPzQX2UBStsIB4WGYmqqW3tV19E8Li
|
27
|
+
bGk5klu+lXK0UomAm2MD3MRR10UCkVFXM4/cUfiMrAgG232yDwRLiGp1EmY3uHyD
|
28
|
+
8/5mRJzBtLsGQKbfBPPNExiFqDzXr2ZwE7tyfsB8auSV3mkVjYjYYFnDfE835U+y
|
29
29
|
-----END CERTIFICATE-----
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
86a8abdffc0dd114d50cb20c7cc635bdb4bdcb16370fdee5aa5c05b4861faacd
|
@@ -1,7 +1,7 @@
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
2
|
-
|
2
|
+
MIIFEzCCAvsCAQEwDQYJKoZIhvcNAQELBQAwTDELMAkGA1UEBhMCUFQxCzAJBgNV
|
3
3
|
BAgMAk5BMQ8wDQYDVQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQD
|
4
|
-
|
4
|
+
DAZSb290Q0EwHhcNMjMwNTMwMTUxMDM4WhcNMjQwNTI5MTUxMDM4WjBTMQswCQYD
|
5
5
|
VQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwF
|
6
6
|
TXlMYWIxFjAUBgNVBAMMDWVsYXN0aWNzZWFyY2gwggIiMA0GCSqGSIb3DQEBAQUA
|
7
7
|
A4ICDwAwggIKAoICAQDGIT9szzhN5HvZ2nivnCDzVfdYbbqBhgEbPppWPyFcV0r2
|
@@ -15,16 +15,16 @@ QcDuDDHfObWhzb4rS55BERIwDUqD1LgCRd0ikRxPSvI1AM4cl35b4DTaDLcnM6EO
|
|
15
15
|
fy+QTYsgNoftU1PI1onDQ7ZdfgrTrIBFQQRwOqfyB4bB2zWVj62LSDvZoYYicNUe
|
16
16
|
cqyE1542WNKzmyE8Mrf3uknN2J6EH7EhmiyRBtGg3NEQCwIYM4/kWPNPOtkSjsn3
|
17
17
|
cNbMNUZiSnQn/nTs4T8g6b2rrwsay/FGUE83AbPqqcTlp2RUVnjbC8KA5+iV1wID
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
18
|
+
AQABMA0GCSqGSIb3DQEBCwUAA4ICAQC4pZQWJoyNANFscsmm6I/u5LgerRCnCS+Y
|
19
|
+
V0tLWribc/iHQNzmUygvwT8+tllp+OzWmp/7oDuFrD/HFf8Xmaj14LTS2QVhZPao
|
20
|
+
NMxkcq7W1WRnTdT+4Do4QE3l5hClgfOvcKzzrhCIPQ//aYgUo5JmXUsYoIWSsJ6v
|
21
|
+
Z77j70xsz95k+C2qfiIKVx6a0U0lXjkp1fwrpGsG0dUL8GAibcX+BYD91uTMTAmg
|
22
|
+
SB4O2xMR6kKCCH3SDuRnkI+iC4MZL64JTrGzn+juMeK0fqZWrdyoVKRlFI92Ao3J
|
23
|
+
oFNF9KTCWQPFAaiy1zl3vDAfIfyJBzQsP6D2BaxVlSQ5jbCxRxOapkhwNUUQ2/vg
|
24
|
+
anggOIOnm6biKhain4U118jviSEGonTvzcIpjExLVXXTF216Ahd7vQWDWqZkYNi3
|
25
|
+
ji1jZSemBWyv/bDxo/STLL/QiX73WuxelerZeMxS03U6KYyhtTLxDvpeOCstbe1u
|
26
|
+
wI9LfpdtAbXP+d/djvYJSHb6WoS4Oya2mnyddqDd6Mfn7lTudhqtw5CLV+sElnFf
|
27
|
+
XDBjDp6tWUin6s/IWPVnvMceZW/sW7oO3sy9NKQx1acHh85WIMrZ+9iVvwcvQMRF
|
28
|
+
p/bTR2VL3jD5fioNyfBmmQ17wIOIm3QvY+nJmzOWalzYYzottPTiabzKYap7Ulcx
|
29
|
+
HDB+ZvuJBA==
|
30
30
|
-----END CERTIFICATE-----
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
b6ee8b4a213328074df1da1f3b8fdde95c5e97e9b1d7da58dad0537a04939b97
|
Binary file
|