logstash-input-elasticsearch 4.23.1 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -18
- data/docs/index.asciidoc +25 -359
- data/lib/logstash/inputs/elasticsearch/aggregation.rb +8 -11
- data/lib/logstash/inputs/elasticsearch/paginated_search.rb +2 -12
- data/lib/logstash/inputs/elasticsearch.rb +50 -237
- data/logstash-input-elasticsearch.gemspec +4 -4
- data/spec/fixtures/test_certs/ca.crt +18 -17
- data/spec/fixtures/test_certs/ca.der.sha256 +1 -1
- data/spec/fixtures/test_certs/es.crt +18 -17
- data/spec/inputs/elasticsearch_spec.rb +28 -293
- data/spec/inputs/integration/elasticsearch_spec.rb +2 -10
- metadata +24 -42
- data/lib/logstash/inputs/elasticsearch/cursor_tracker.rb +0 -58
- data/lib/logstash/inputs/elasticsearch/esql.rb +0 -153
- data/spec/fixtures/test_certs/GENERATED_AT +0 -1
- data/spec/fixtures/test_certs/es.chain.crt +0 -38
- data/spec/fixtures/test_certs/renew.sh +0 -15
- data/spec/inputs/cursor_tracker_spec.rb +0 -72
- data/spec/inputs/elasticsearch_esql_spec.rb +0 -180
- data/spec/inputs/integration/elasticsearch_esql_spec.rb +0 -150
- data/version +0 -1
|
@@ -13,7 +13,9 @@ require "logstash/plugin_mixins/normalize_config_support"
|
|
|
13
13
|
require "base64"
|
|
14
14
|
|
|
15
15
|
require "elasticsearch"
|
|
16
|
-
require "manticore"
|
|
16
|
+
require "elasticsearch/transport/transport/http/manticore"
|
|
17
|
+
require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
|
|
18
|
+
require_relative "elasticsearch/patches/_elasticsearch_transport_connections_selector"
|
|
17
19
|
|
|
18
20
|
# .Compatibility Note
|
|
19
21
|
# [NOTE]
|
|
@@ -73,8 +75,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
73
75
|
|
|
74
76
|
require 'logstash/inputs/elasticsearch/paginated_search'
|
|
75
77
|
require 'logstash/inputs/elasticsearch/aggregation'
|
|
76
|
-
require 'logstash/inputs/elasticsearch/cursor_tracker'
|
|
77
|
-
require 'logstash/inputs/elasticsearch/esql'
|
|
78
78
|
|
|
79
79
|
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
|
80
80
|
include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
|
|
@@ -97,21 +97,15 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
97
97
|
# The index or alias to search.
|
|
98
98
|
config :index, :validate => :string, :default => "logstash-*"
|
|
99
99
|
|
|
100
|
-
#
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
# The query to be executed. DSL or ES|QL (when `query_type => 'esql'`) query shape is accepted.
|
|
104
|
-
# Read the following documentations for more info
|
|
105
|
-
# Query DSL: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
|
106
|
-
# ES|QL: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html
|
|
100
|
+
# The query to be executed. Read the Elasticsearch query DSL documentation
|
|
101
|
+
# for more info
|
|
102
|
+
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
|
107
103
|
config :query, :validate => :string, :default => '{ "sort": [ "_doc" ] }'
|
|
108
104
|
|
|
109
|
-
# This allows you to
|
|
110
|
-
# where
|
|
111
|
-
#
|
|
112
|
-
|
|
113
|
-
# Note that this param is invalid when `query_type => 'esql'`, ES|QL response shape is always a tabular format
|
|
114
|
-
config :response_type, :validate => %w[hits aggregations], :default => 'hits'
|
|
105
|
+
# This allows you to speccify the response type: either hits or aggregations
|
|
106
|
+
# where hits: normal search request
|
|
107
|
+
# aggregations: aggregation request
|
|
108
|
+
config :response_type, :validate => ['hits', 'aggregations'], :default => 'hits'
|
|
115
109
|
|
|
116
110
|
# This allows you to set the maximum number of hits returned per scroll.
|
|
117
111
|
config :size, :validate => :number, :default => 1000
|
|
@@ -132,20 +126,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
132
126
|
# by this pipeline input.
|
|
133
127
|
config :slices, :validate => :number
|
|
134
128
|
|
|
135
|
-
# Enable tracking the value of a given field to be used as a cursor
|
|
136
|
-
# Main concerns:
|
|
137
|
-
# * using anything other than _event.timestamp easily leads to data loss
|
|
138
|
-
# * the first "synchronization run can take a long time"
|
|
139
|
-
config :tracking_field, :validate => :string
|
|
140
|
-
|
|
141
|
-
# Define the initial seed value of the tracking_field
|
|
142
|
-
config :tracking_field_seed, :validate => :string, :default => "1970-01-01T00:00:00.000000000Z"
|
|
143
|
-
|
|
144
|
-
# The location of where the tracking field value will be stored
|
|
145
|
-
# The value is persisted after each scheduled run (and not per result)
|
|
146
|
-
# If it's not set it defaults to '${path.data}/plugins/inputs/elasticsearch/<pipeline_id>/last_run_value'
|
|
147
|
-
config :last_run_metadata_path, :validate => :string
|
|
148
|
-
|
|
149
129
|
# If set, include Elasticsearch document information such as index, type, and
|
|
150
130
|
# the id in the event.
|
|
151
131
|
#
|
|
@@ -215,30 +195,18 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
215
195
|
config :cloud_auth, :validate => :password
|
|
216
196
|
|
|
217
197
|
# Authenticate using Elasticsearch API key.
|
|
218
|
-
#
|
|
219
|
-
# its base64-encoded form, or an https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key] (prefixed with `essu_`) can be used.
|
|
198
|
+
# format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
|
|
220
199
|
config :api_key, :validate => :password
|
|
221
200
|
|
|
222
201
|
# Set the address of a forward HTTP proxy.
|
|
223
202
|
config :proxy, :validate => :uri_or_empty
|
|
224
203
|
|
|
225
|
-
# SSL
|
|
226
|
-
config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead."
|
|
227
|
-
|
|
228
|
-
# SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary
|
|
229
|
-
config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead."
|
|
230
|
-
|
|
231
204
|
# OpenSSL-style X.509 certificate certificate to authenticate the client
|
|
232
205
|
config :ssl_certificate, :validate => :path
|
|
233
206
|
|
|
234
207
|
# SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary
|
|
235
208
|
config :ssl_certificate_authorities, :validate => :path, :list => true
|
|
236
209
|
|
|
237
|
-
# Option to validate the server's certificate. Disabling this severely compromises security.
|
|
238
|
-
# For more information on the importance of certificate verification please read
|
|
239
|
-
# https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
|
|
240
|
-
config :ssl_certificate_verification, :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead."
|
|
241
|
-
|
|
242
210
|
# The list of cipher suites to use, listed by priorities.
|
|
243
211
|
# Supported cipher suites vary depending on which version of Java is used.
|
|
244
212
|
config :ssl_cipher_suites, :validate => :string, :list => true
|
|
@@ -266,7 +234,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
266
234
|
config :ssl_truststore_password, :validate => :password
|
|
267
235
|
|
|
268
236
|
# The JKS truststore to validate the server's certificate.
|
|
269
|
-
# Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
|
|
270
237
|
config :ssl_truststore_path, :validate => :path
|
|
271
238
|
|
|
272
239
|
# The format of the truststore file. It must be either jks or pkcs12
|
|
@@ -285,13 +252,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
285
252
|
# exactly once.
|
|
286
253
|
config :schedule, :validate => :string
|
|
287
254
|
|
|
288
|
-
# Allow scheduled runs to overlap (enabled by default). Setting to false will
|
|
289
|
-
# only start a new scheduled run after the previous one completes.
|
|
290
|
-
config :schedule_overlap, :validate => :boolean
|
|
291
|
-
|
|
292
255
|
# If set, the _source of each hit will be added nested under the target instead of at the top-level
|
|
293
256
|
config :target, :validate => :field_reference
|
|
294
257
|
|
|
258
|
+
# Obsolete Settings
|
|
259
|
+
config :ssl, :obsolete => "Set 'ssl_enabled' instead."
|
|
260
|
+
config :ca_file, :obsolete => "Set 'ssl_certificate_authorities' instead."
|
|
261
|
+
config :ssl_certificate_verification, :obsolete => "Set 'ssl_verification_mode' instead."
|
|
262
|
+
|
|
295
263
|
# config :ca_trusted_fingerprint, :validate => :sha_256_hex
|
|
296
264
|
include LogStash::PluginMixins::CATrustedFingerprintSupport
|
|
297
265
|
|
|
@@ -301,9 +269,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
301
269
|
DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
|
|
302
270
|
INTERNAL_ORIGIN_HEADER = { 'x-elastic-product-origin' => 'logstash-input-elasticsearch'}.freeze
|
|
303
271
|
|
|
304
|
-
LS_ESQL_SUPPORT_VERSION = "8.17.4" # the version started using elasticsearch-ruby v8
|
|
305
|
-
ES_ESQL_SUPPORT_VERSION = "8.11.0"
|
|
306
|
-
|
|
307
272
|
def initialize(params={})
|
|
308
273
|
super(params)
|
|
309
274
|
|
|
@@ -320,17 +285,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
320
285
|
fill_hosts_from_cloud_id
|
|
321
286
|
setup_ssl_params!
|
|
322
287
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
raise(LogStash::ConfigurationError, "Configured #{not_allowed_options} params are not allowed while using ES|QL query") if not_allowed_options&.size > 1
|
|
328
|
-
else
|
|
329
|
-
@base_query = LogStash::Json.load(@query)
|
|
330
|
-
if @slices
|
|
331
|
-
@base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
|
|
332
|
-
@slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
|
|
333
|
-
end
|
|
288
|
+
@base_query = LogStash::Json.load(@query)
|
|
289
|
+
if @slices
|
|
290
|
+
@base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
|
|
291
|
+
@slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
|
|
334
292
|
end
|
|
335
293
|
|
|
336
294
|
@retries < 0 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `retries` option must be equal or greater than zero, got `#{@retries}`")
|
|
@@ -358,7 +316,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
358
316
|
@client_options = {
|
|
359
317
|
:hosts => hosts,
|
|
360
318
|
:transport_options => transport_options,
|
|
361
|
-
:transport_class =>
|
|
319
|
+
:transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore,
|
|
362
320
|
:ssl => ssl_options
|
|
363
321
|
}
|
|
364
322
|
|
|
@@ -366,27 +324,21 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
366
324
|
|
|
367
325
|
test_connection!
|
|
368
326
|
|
|
369
|
-
validate_es_for_esql_support!
|
|
370
|
-
|
|
371
327
|
setup_serverless
|
|
372
328
|
|
|
373
329
|
setup_search_api
|
|
374
330
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
setup_cursor_tracker
|
|
331
|
+
setup_query_executor
|
|
378
332
|
|
|
379
333
|
@client
|
|
380
334
|
end
|
|
381
335
|
|
|
382
336
|
def run(output_queue)
|
|
383
337
|
if @schedule
|
|
384
|
-
scheduler.cron(@schedule
|
|
385
|
-
@query_executor.do_run(output_queue, get_query_object())
|
|
386
|
-
end
|
|
338
|
+
scheduler.cron(@schedule) { @query_executor.do_run(output_queue) }
|
|
387
339
|
scheduler.join
|
|
388
340
|
else
|
|
389
|
-
@query_executor.do_run(output_queue
|
|
341
|
+
@query_executor.do_run(output_queue)
|
|
390
342
|
end
|
|
391
343
|
end
|
|
392
344
|
|
|
@@ -394,42 +346,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
394
346
|
# This can be called externally from the query_executor
|
|
395
347
|
public
|
|
396
348
|
def push_hit(hit, output_queue, root_field = '_source')
|
|
397
|
-
event = event_from_hit(hit, root_field)
|
|
398
|
-
decorate(event)
|
|
399
|
-
output_queue << event
|
|
400
|
-
record_last_value(event)
|
|
401
|
-
end
|
|
402
|
-
|
|
403
|
-
def decorate_event(event)
|
|
404
|
-
decorate(event)
|
|
405
|
-
end
|
|
406
|
-
|
|
407
|
-
private
|
|
408
|
-
|
|
409
|
-
def get_query_object
|
|
410
|
-
return @query if @query_type == 'esql'
|
|
411
|
-
if @cursor_tracker
|
|
412
|
-
query = @cursor_tracker.inject_cursor(@query)
|
|
413
|
-
@logger.debug("new query is #{query}")
|
|
414
|
-
else
|
|
415
|
-
query = @query
|
|
416
|
-
end
|
|
417
|
-
LogStash::Json.load(query)
|
|
418
|
-
end
|
|
419
|
-
|
|
420
|
-
def record_last_value(event)
|
|
421
|
-
@cursor_tracker.record_last_value(event) if @tracking_field
|
|
422
|
-
end
|
|
423
|
-
|
|
424
|
-
def event_from_hit(hit, root_field)
|
|
425
349
|
event = targeted_event_factory.new_event hit[root_field]
|
|
426
350
|
set_docinfo_fields(hit, event) if @docinfo
|
|
427
|
-
|
|
428
|
-
event
|
|
429
|
-
rescue => e
|
|
430
|
-
serialized_hit = hit.to_json
|
|
431
|
-
logger.warn("Event creation error, original data now in [event][original] field", message: e.message, exception: e.class, data: serialized_hit)
|
|
432
|
-
return event_factory.new_event('event' => { 'original' => serialized_hit }, 'tags' => ['_elasticsearch_input_failure'])
|
|
351
|
+
decorate(event)
|
|
352
|
+
output_queue << event
|
|
433
353
|
end
|
|
434
354
|
|
|
435
355
|
def set_docinfo_fields(hit, event)
|
|
@@ -437,8 +357,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
437
357
|
docinfo_target = event.get(@docinfo_target) || {}
|
|
438
358
|
|
|
439
359
|
unless docinfo_target.is_a?(Hash)
|
|
440
|
-
|
|
441
|
-
|
|
360
|
+
@logger.error("Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event.to_hash_with_metadata)
|
|
361
|
+
|
|
362
|
+
# TODO: (colin) I am not sure raising is a good strategy here?
|
|
363
|
+
raise Exception.new("Elasticsearch input: incompatible event")
|
|
442
364
|
end
|
|
443
365
|
|
|
444
366
|
@docinfo_fields.each do |field|
|
|
@@ -448,6 +370,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
448
370
|
event.set(@docinfo_target, docinfo_target)
|
|
449
371
|
end
|
|
450
372
|
|
|
373
|
+
private
|
|
374
|
+
|
|
451
375
|
def hosts_default?(hosts)
|
|
452
376
|
hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
|
|
453
377
|
end
|
|
@@ -481,8 +405,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
481
405
|
ssl_options[:ssl] = true if @ssl_enabled
|
|
482
406
|
|
|
483
407
|
unless @ssl_enabled
|
|
484
|
-
# Keep it backward compatible with the deprecated `ssl` option
|
|
485
|
-
ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint if original_params.include?('ssl')
|
|
486
408
|
return ssl_options
|
|
487
409
|
end
|
|
488
410
|
|
|
@@ -546,38 +468,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
546
468
|
end
|
|
547
469
|
|
|
548
470
|
def setup_ssl_params!
|
|
549
|
-
|
|
550
|
-
|
|
471
|
+
# Only infer ssl_enabled if it wasn't explicitly set
|
|
472
|
+
unless original_params.include?('ssl_enabled')
|
|
473
|
+
@ssl_enabled = effectively_ssl?
|
|
474
|
+
params['ssl_enabled'] = @ssl_enabled
|
|
551
475
|
end
|
|
552
|
-
|
|
553
|
-
# Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set
|
|
554
|
-
infer_ssl_enabled_from_hosts
|
|
555
|
-
|
|
556
|
-
@ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
|
|
557
|
-
normalize.with_deprecated_mapping(:ca_file) do |ca_file|
|
|
558
|
-
[ca_file]
|
|
559
|
-
end
|
|
560
|
-
end
|
|
561
|
-
|
|
562
|
-
@ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize|
|
|
563
|
-
normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification|
|
|
564
|
-
if ssl_certificate_verification == true
|
|
565
|
-
"full"
|
|
566
|
-
else
|
|
567
|
-
"none"
|
|
568
|
-
end
|
|
569
|
-
end
|
|
570
|
-
end
|
|
571
|
-
|
|
572
|
-
params['ssl_enabled'] = @ssl_enabled
|
|
573
|
-
params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
|
|
574
|
-
params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
|
|
575
|
-
end
|
|
576
|
-
|
|
577
|
-
def infer_ssl_enabled_from_hosts
|
|
578
|
-
return if original_params.include?('ssl') || original_params.include?('ssl_enabled')
|
|
579
|
-
|
|
580
|
-
@ssl_enabled = params['ssl_enabled'] = effectively_ssl?
|
|
581
476
|
end
|
|
582
477
|
|
|
583
478
|
def setup_hosts
|
|
@@ -600,40 +495,12 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
600
495
|
end
|
|
601
496
|
|
|
602
497
|
def setup_api_key(api_key)
|
|
603
|
-
return {} unless (api_key
|
|
498
|
+
return {} unless (api_key && api_key.value)
|
|
604
499
|
|
|
605
|
-
token =
|
|
500
|
+
token = ::Base64.strict_encode64(api_key.value)
|
|
606
501
|
{ 'Authorization' => "ApiKey #{token}" }
|
|
607
502
|
end
|
|
608
503
|
|
|
609
|
-
# Resolves the `api_key` value into the credential used in the
|
|
610
|
-
# `Authorization: ApiKey` header. An already base64-encoded key and an Elastic
|
|
611
|
-
# Cloud API key are used as-is; a raw `id:api_key` pair is base64-encoded. An
|
|
612
|
-
# unrecognized value is rejected so a malformed key surfaces at startup rather
|
|
613
|
-
# than as a later authentication failure.
|
|
614
|
-
def resolve_api_key(key_value)
|
|
615
|
-
if base64?(key_value) || cloud_api_key?(key_value)
|
|
616
|
-
key_value
|
|
617
|
-
elsif key_value.match?(/\A[^:]+:[^:]+\z/)
|
|
618
|
-
Base64.strict_encode64(key_value)
|
|
619
|
-
else
|
|
620
|
-
raise LogStash::ConfigurationError, "Invalid api_key format. Expected a base64-encoded key, an 'id:api_key' pair, or a Cloud API key (essu_ prefix)."
|
|
621
|
-
end
|
|
622
|
-
end
|
|
623
|
-
|
|
624
|
-
# Elastic Cloud API keys (such as the unified Serverless keys) are opaque
|
|
625
|
-
# tokens prefixed with `essu_` that Elasticsearch accepts verbatim in the
|
|
626
|
-
# `Authorization: ApiKey` header, with no base64 encoding.
|
|
627
|
-
def cloud_api_key?(string)
|
|
628
|
-
string.match?(/\Aessu_.+/)
|
|
629
|
-
end
|
|
630
|
-
|
|
631
|
-
def base64?(string)
|
|
632
|
-
string == Base64.strict_encode64(Base64.strict_decode64(string))
|
|
633
|
-
rescue ArgumentError
|
|
634
|
-
false
|
|
635
|
-
end
|
|
636
|
-
|
|
637
504
|
def prepare_user_agent
|
|
638
505
|
os_name = java.lang.System.getProperty('os.name')
|
|
639
506
|
os_version = java.lang.System.getProperty('os.version')
|
|
@@ -753,72 +620,18 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
|
753
620
|
|
|
754
621
|
end
|
|
755
622
|
|
|
756
|
-
def
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
def setup_cursor_tracker
|
|
769
|
-
return unless @tracking_field
|
|
770
|
-
return unless @query_executor.is_a?(LogStash::Inputs::Elasticsearch::SearchAfter)
|
|
771
|
-
|
|
772
|
-
if @resolved_search_api != "search_after" || @response_type != "hits"
|
|
773
|
-
raise ConfigurationError.new("The `tracking_field` feature can only be used with `search_after` non-aggregation queries")
|
|
774
|
-
end
|
|
775
|
-
|
|
776
|
-
@cursor_tracker = CursorTracker.new(last_run_metadata_path: last_run_metadata_path,
|
|
777
|
-
tracking_field: @tracking_field,
|
|
778
|
-
tracking_field_seed: @tracking_field_seed)
|
|
779
|
-
@query_executor.cursor_tracker = @cursor_tracker
|
|
780
|
-
end
|
|
781
|
-
|
|
782
|
-
def last_run_metadata_path
|
|
783
|
-
return @last_run_metadata_path if @last_run_metadata_path
|
|
784
|
-
|
|
785
|
-
last_run_metadata_path = ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "elasticsearch", pipeline_id, "last_run_value")
|
|
786
|
-
FileUtils.mkdir_p ::File.dirname(last_run_metadata_path)
|
|
787
|
-
last_run_metadata_path
|
|
788
|
-
end
|
|
789
|
-
|
|
790
|
-
def get_transport_client_class
|
|
791
|
-
# LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport`
|
|
792
|
-
# And now `elasticsearch-transport` is old, instead we have `elastic-transport`.
|
|
793
|
-
# LS-core updated `elasticsearch` > 8: https://github.com/elastic/logstash/pull/17161
|
|
794
|
-
# Following source bits are for the compatibility to support both `elasticsearch-transport` and `elastic-transport` gems
|
|
795
|
-
require "elasticsearch/transport/transport/http/manticore"
|
|
796
|
-
require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
|
|
797
|
-
require_relative "elasticsearch/patches/_elasticsearch_transport_connections_selector"
|
|
798
|
-
::Elasticsearch::Transport::Transport::HTTP::Manticore
|
|
799
|
-
rescue ::LoadError
|
|
800
|
-
require "elastic/transport/transport/http/manticore"
|
|
801
|
-
::Elastic::Transport::Transport::HTTP::Manticore
|
|
802
|
-
end
|
|
803
|
-
|
|
804
|
-
def validate_ls_version_for_esql_support!
|
|
805
|
-
if Gem::Version.create(LOGSTASH_VERSION) < Gem::Version.create(LS_ESQL_SUPPORT_VERSION)
|
|
806
|
-
fail("Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{LS_ESQL_SUPPORT_VERSION}")
|
|
807
|
-
end
|
|
808
|
-
end
|
|
809
|
-
|
|
810
|
-
def validate_esql_query!
|
|
811
|
-
fail(LogStash::ConfigurationError, "`query` cannot be empty") if @query.strip.empty?
|
|
812
|
-
source_commands = %w[FROM ROW SHOW]
|
|
813
|
-
contains_source_command = source_commands.any? { |source_command| @query.strip.start_with?(source_command) }
|
|
814
|
-
fail(LogStash::ConfigurationError, "`query` needs to start with any of #{source_commands}") unless contains_source_command
|
|
815
|
-
end
|
|
816
|
-
|
|
817
|
-
def validate_es_for_esql_support!
|
|
818
|
-
return unless @query_type == 'esql'
|
|
819
|
-
# make sure connected ES supports ES|QL (8.11+)
|
|
820
|
-
es_supports_esql = Gem::Version.create(es_version) >= Gem::Version.create(ES_ESQL_SUPPORT_VERSION)
|
|
821
|
-
fail("Connected Elasticsearch #{es_version} version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{ES_ESQL_SUPPORT_VERSION} version.") unless es_supports_esql
|
|
623
|
+
def setup_query_executor
|
|
624
|
+
@query_executor = case @response_type
|
|
625
|
+
when 'hits'
|
|
626
|
+
if @resolved_search_api == "search_after"
|
|
627
|
+
LogStash::Inputs::Elasticsearch::SearchAfter.new(@client, self)
|
|
628
|
+
else
|
|
629
|
+
logger.warn("scroll API is no longer recommended for pagination. Consider using search_after instead.") if es_major_version >= 8
|
|
630
|
+
LogStash::Inputs::Elasticsearch::Scroll.new(@client, self)
|
|
631
|
+
end
|
|
632
|
+
when 'aggregations'
|
|
633
|
+
LogStash::Inputs::Elasticsearch::Aggregation.new(@client, self)
|
|
634
|
+
end
|
|
822
635
|
end
|
|
823
636
|
|
|
824
637
|
module URIOrEmptyValidator
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
|
4
|
-
s.version =
|
|
4
|
+
s.version = '5.0.0'
|
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
|
8
8
|
s.authors = ["Elastic"]
|
|
9
9
|
s.email = 'info@elastic.co'
|
|
10
|
-
s.homepage = "
|
|
10
|
+
s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
|
|
11
11
|
s.require_paths = ["lib"]
|
|
12
12
|
|
|
13
13
|
# Files
|
|
14
|
-
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "
|
|
14
|
+
s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
|
|
15
15
|
|
|
16
16
|
# Tests
|
|
17
17
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
|
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
|
|
|
26
26
|
s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
|
|
27
27
|
s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
|
|
28
28
|
|
|
29
|
-
s.add_runtime_dependency 'elasticsearch', '>= 7.17.9'
|
|
29
|
+
s.add_runtime_dependency 'elasticsearch', '>= 7.17.9'
|
|
30
30
|
s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
|
|
31
31
|
s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
|
|
32
32
|
|
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
2
|
+
MIIDSTCCAjGgAwIBAgIUUcAg9c8B8jiliCkOEJyqoAHrmccwDQYJKoZIhvcNAQEL
|
|
3
|
+
BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
|
|
4
|
+
cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNDU1WhcNMjQwODExMDUxNDU1WjA0MTIwMAYD
|
|
5
|
+
VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
|
|
6
|
+
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK1HuusRuGNsztd4EQvqwcMr
|
|
7
|
+
8XvnNNaalerpMOorCGySEFrNf0HxDIVMGMCrOv1F8SvlcGq3XANs2MJ4F2xhhLZr
|
|
8
|
+
PpqVHx+QnSZ66lu5R89QVSuMh/dCMxhNBlOA/dDlvy+EJBl9H791UGy/ChhSgaBd
|
|
9
|
+
OKVyGkhjErRTeMIq7rR7UG6GL/fV+JGy41UiLrm1KQP7/XVD9UzZfGq/hylFkTPe
|
|
10
|
+
oox5BUxdxUdDZ2creOID+agtIYuJVIkelKPQ+ljBY3kWBRexqJQsvyNUs1gZpjpz
|
|
11
|
+
YUCzuVcXDRuJXYQXGqWXhsBPfJv+ZcSyMIBUfWT/G13cWU1iwufPy0NjajowPZsC
|
|
12
|
+
AwEAAaNTMFEwHQYDVR0OBBYEFMgkye5+2l+TE0I6RsXRHjGBwpBGMB8GA1UdIwQY
|
|
13
|
+
MBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
|
|
14
|
+
hvcNAQELBQADggEBAIgtJW8sy5lBpzPRHkmWSS/SCZIPsABW+cHqQ3e0udrI3CLB
|
|
15
|
+
G9n7yqAPWOBTbdqC2GM8dvAS/Twx4Bub/lWr84dFCu+t0mQq4l5kpJMVRS0KKXPL
|
|
16
|
+
DwJbUN3oPNYy4uPn5Xi+XY3BYFce5vwJUsqIxeAbIOxVTNx++k5DFnB0ESAM23QL
|
|
17
|
+
sgUZl7xl3/DkdO4oHj30gmTRW9bjCJ6umnHIiO3JoJatrprurUIt80vHC4Ndft36
|
|
18
|
+
NBQ9mZpequ4RYjpSZNLcVsxyFAYwEY4g8MvH0MoMo2RRLfehmMCzXnI/Wh2qEyYz
|
|
19
|
+
emHprBii/5y1HieKXlX9CZRb5qEPHckDVXW3znw=
|
|
19
20
|
-----END CERTIFICATE-----
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
195a7e7b1bc29f3d7913a918a44721704d27fa56facea0cd72a8093c7107c283
|
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
-----BEGIN CERTIFICATE-----
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
/
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
2
|
+
MIIDNjCCAh6gAwIBAgIUF9wE+oqGSbm4UVn1y9gEjzyaJFswDQYJKoZIhvcNAQEL
|
|
3
|
+
BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
|
|
4
|
+
cmF0ZWQgQ0EwHhcNMjEwODEyMDUxNTI3WhcNMjQwODExMDUxNTI3WjANMQswCQYD
|
|
5
|
+
VQQDEwJlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK2S2by0lgyu
|
|
6
|
+
1JfgGgZ41PNXbH2qMPMzowguVVdtZ16WM0CaEG7lnLxmMcC+2Q7NnGuFnPAVQo9T
|
|
7
|
+
Q3bh7j+1PkCJVHUKZfJIeWtGc9+qXBcO1MhedfwM1osSa4bfwM85G+XKWbRNtmSt
|
|
8
|
+
CoUuKArIyZkzdBAAQLBoQyPf3DIza1Au4j9Hb3zrswD6e7n2PN4ffIyil1GFduLJ
|
|
9
|
+
2275qqFiOhkEDUhv7BKNftVBh/89O/5lSqAQGuQ1aDRr8TdHwhO71u4ZIU/Pn6yX
|
|
10
|
+
LGBWrQG53+qpdCsxGvJTfbtIEYUDTN83CirIxDKJgc1QXOEldylztHf4xnQ7ZarJ
|
|
11
|
+
tqF6pUzHbRsCAwEAAaNnMGUwHQYDVR0OBBYEFFQUK+6Cg2kExRj1xSDzEi4kkgKX
|
|
12
|
+
MB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGBwpBGMBgGA1UdEQQRMA+CDWVs
|
|
13
|
+
YXN0aWNzZWFyY2gwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAQEAinaknZIc
|
|
14
|
+
7xtQNwUwa+kdET+I4lMz+TJw9vTjGKPJqe082n81ycKU5b+a/OndG90z+dTwhShW
|
|
15
|
+
f0oZdIe/1rDCdiRU4ceCZA4ybKrFDIbW8gOKZOx9rsgEx9XNELj4ocZTBqxjQmNE
|
|
16
|
+
Ho91fli5aEm0EL2vJgejh4hcfDeElQ6go9gtvAHQ57XEADQSenvt69jOICOupnS+
|
|
17
|
+
LSjDVhv/VLi3CAip0B+lD5fX/DVQdrJ62eRGuQYxoouE3saCO58qUUrKB39yD9KA
|
|
18
|
+
qRA/sVxyLogxaU+5dLfc0NJdOqSzStxQ2vdMvAWo9tZZ2UBGFrk5SdwCQe7Yv5mX
|
|
19
|
+
qi02i4q6meHGcw==
|
|
19
20
|
-----END CERTIFICATE-----
|