logstash-input-elasticsearch 4.17.0 → 4.17.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/docs/index.asciidoc +1 -0
- data/lib/logstash/inputs/elasticsearch.rb +19 -39
- data/logstash-input-elasticsearch.gemspec +1 -1
- data/spec/inputs/elasticsearch_spec.rb +20 -15
- data/spec/inputs/elasticsearch_ssl_spec.rb +3 -3
- data/spec/inputs/integration/elasticsearch_spec.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 54b223c24702d4e9eb27efc05d351dbe9138d7be576a72e552bea3ef40254bb5
|
4
|
+
data.tar.gz: 697ade265dd83a9b87a6802e650a9100e64195315577e927c8228cdb6b18e720
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 50ac2baa3e825cca7ffd51321bf42530b5c6f32dd88db94c9a0ce5ec2e978de5e0e82f645d6c8116972b40e6efae01782a73dc38bbda50d513df1bb88acf6e75
|
7
|
+
data.tar.gz: 50290a768d6f8985cef40b393bdf4d80842416825c5d3e8d835485b9a96adf4f52cbfd654fb687c92cc22fdcc616cf70147f0d9ded807e1a07d135f23e6a2fd7
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 4.17.2
|
2
|
+
- Fixes a regression introduced in 4.17.0 which could prevent a connection from being established to Elasticsearch in some SSL configurations [#193](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/193)
|
3
|
+
|
4
|
+
## 4.17.1
|
5
|
+
- Fix: scroll slice high memory consumption [#189](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/189)
|
6
|
+
|
1
7
|
## 4.17.0
|
2
8
|
- Added SSL settings for: [#185](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/185)
|
3
9
|
- `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme
|
data/docs/index.asciidoc
CHANGED
@@ -353,6 +353,7 @@ documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a l
|
|
353
353
|
The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message.
|
354
354
|
The default is 0 (no retry). This value should be equal to or greater than zero.
|
355
355
|
|
356
|
+
NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible.
|
356
357
|
|
357
358
|
[id="plugins-{type}s-{plugin}-schedule"]
|
358
359
|
===== `schedule`
|
@@ -256,6 +256,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
256
256
|
# config :ca_trusted_fingerprint, :validate => :sha_256_hex
|
257
257
|
include LogStash::PluginMixins::CATrustedFingerprintSupport
|
258
258
|
|
259
|
+
attr_reader :pipeline_id
|
260
|
+
|
259
261
|
def initialize(params={})
|
260
262
|
super(params)
|
261
263
|
|
@@ -267,6 +269,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
267
269
|
def register
|
268
270
|
require "rufus/scheduler"
|
269
271
|
|
272
|
+
@pipeline_id = execution_context&.pipeline_id || 'main'
|
273
|
+
|
270
274
|
fill_hosts_from_cloud_id
|
271
275
|
setup_ssl_params!
|
272
276
|
|
@@ -326,20 +330,22 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
326
330
|
def do_run(output_queue)
|
327
331
|
# if configured to run a single slice, don't bother spinning up threads
|
328
332
|
if @slices.nil? || @slices <= 1
|
329
|
-
|
330
|
-
|
331
|
-
|
333
|
+
return retryable(JOB_NAME) do
|
334
|
+
do_run_slice(output_queue)
|
335
|
+
end
|
332
336
|
end
|
333
337
|
|
334
338
|
logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
|
335
339
|
|
336
|
-
slice_results = parallel_slice # array of tuple(ok, events)
|
337
340
|
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
341
|
+
@slices.times.map do |slice_id|
|
342
|
+
Thread.new do
|
343
|
+
LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
|
344
|
+
retryable(JOB_NAME) do
|
345
|
+
do_run_slice(output_queue, slice_id)
|
346
|
+
end
|
347
|
+
end
|
348
|
+
end.map(&:join)
|
343
349
|
|
344
350
|
logger.trace("#{@slices} slices completed")
|
345
351
|
end
|
@@ -347,42 +353,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
347
353
|
def retryable(job_name, &block)
|
348
354
|
begin
|
349
355
|
stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
|
350
|
-
|
351
|
-
[true, output]
|
356
|
+
stud_try.try((@retries + 1).times) { yield }
|
352
357
|
rescue => e
|
353
358
|
error_details = {:message => e.message, :cause => e.cause}
|
354
359
|
error_details[:backtrace] = e.backtrace if logger.debug?
|
355
360
|
logger.error("Tried #{job_name} unsuccessfully", error_details)
|
356
|
-
[false, nil]
|
357
361
|
end
|
358
362
|
end
|
359
363
|
|
360
|
-
|
361
|
-
# @return [(ok, events)] : Array of tuple(Boolean, [Logstash::Event])
|
362
|
-
def parallel_slice
|
363
|
-
pipeline_id = execution_context&.pipeline_id || 'main'
|
364
|
-
@slices.times.map do |slice_id|
|
365
|
-
Thread.new do
|
366
|
-
LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
|
367
|
-
retryable_slice(slice_id)
|
368
|
-
end
|
369
|
-
end.map do |t|
|
370
|
-
t.join
|
371
|
-
t.value
|
372
|
-
end
|
373
|
-
end
|
374
|
-
|
375
|
-
# @param scroll_id [Integer]
|
376
|
-
# @return (ok, events) [Boolean, Array(Logstash::Event)]
|
377
|
-
def retryable_slice(slice_id=nil)
|
378
|
-
retryable(JOB_NAME) do
|
379
|
-
output = []
|
380
|
-
do_run_slice(output, slice_id)
|
381
|
-
output
|
382
|
-
end
|
383
|
-
end
|
384
|
-
|
385
|
-
|
386
364
|
def do_run_slice(output_queue, slice_id=nil)
|
387
365
|
slice_query = @base_query
|
388
366
|
slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
|
@@ -543,7 +521,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
543
521
|
"to make sure your data is secure set `ssl_verification_mode => full`"
|
544
522
|
ssl_options[:verify] = :disable
|
545
523
|
else
|
546
|
-
|
524
|
+
# Manticore's :default maps to Apache HTTP Client's DefaultHostnameVerifier,
|
525
|
+
# which is the modern STRICT verifier that replaces the deprecated StrictHostnameVerifier
|
526
|
+
ssl_options[:verify] = :default
|
547
527
|
end
|
548
528
|
end
|
549
529
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '4.17.
|
4
|
+
s.version = '4.17.2'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -204,7 +204,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
204
204
|
context 'with `slices => 1`' do
|
205
205
|
let(:slices) { 1 }
|
206
206
|
it 'runs just one slice' do
|
207
|
-
expect(plugin).to receive(:do_run_slice).with(duck_type(:<<)
|
207
|
+
expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
|
208
208
|
expect(Thread).to_not receive(:new)
|
209
209
|
|
210
210
|
plugin.register
|
@@ -215,7 +215,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
215
215
|
context 'without slices directive' do
|
216
216
|
let(:config) { super().tap { |h| h.delete('slices') } }
|
217
217
|
it 'runs just one slice' do
|
218
|
-
expect(plugin).to receive(:do_run_slice).with(duck_type(:<<)
|
218
|
+
expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
|
219
219
|
expect(Thread).to_not receive(:new)
|
220
220
|
|
221
221
|
plugin.register
|
@@ -414,18 +414,19 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
414
414
|
expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
|
415
415
|
plugin.register
|
416
416
|
|
417
|
-
expect(client).to receive(:clear_scroll).and_return(nil)
|
417
|
+
expect(client).to receive(:clear_scroll).twice.and_return(nil)
|
418
418
|
|
419
|
-
# SLICE0 is a three-page scroll
|
419
|
+
# SLICE0 is a three-page scroll
|
420
420
|
slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
|
421
421
|
expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
|
422
|
-
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).
|
422
|
+
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
|
423
|
+
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
|
423
424
|
allow(client).to receive(:ping)
|
424
425
|
|
425
|
-
# SLICE1 is a two-page scroll in which the last page
|
426
|
+
# SLICE1 is a two-page scroll in which the last page throws exception
|
426
427
|
slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
|
427
428
|
expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
|
428
|
-
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).
|
429
|
+
expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_raise("boom")
|
429
430
|
|
430
431
|
synchronize_method!(plugin, :scroll_request)
|
431
432
|
synchronize_method!(plugin, :search_request)
|
@@ -433,18 +434,22 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
|
|
433
434
|
|
434
435
|
let(:client) { Elasticsearch::Client.new }
|
435
436
|
|
436
|
-
it '
|
437
|
-
expect(plugin).to receive(:
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
437
|
+
it 'insert event to queue without waiting other slices' do
|
438
|
+
expect(plugin).to receive(:do_run_slice).twice.and_wrap_original do |m, *args|
|
439
|
+
q = args[0]
|
440
|
+
slice_id = args[1]
|
441
|
+
if slice_id == 0
|
442
|
+
m.call(*args)
|
443
|
+
expect(q.size).to eq(3)
|
444
|
+
else
|
445
|
+
sleep(1)
|
446
|
+
m.call(*args)
|
447
|
+
end
|
443
448
|
end
|
444
449
|
|
445
450
|
queue = Queue.new
|
446
451
|
plugin.run(queue)
|
447
|
-
expect(queue.size).to eq(
|
452
|
+
expect(queue.size).to eq(5)
|
448
453
|
end
|
449
454
|
end
|
450
455
|
end
|
@@ -123,7 +123,7 @@ describe "SSL options" do
|
|
123
123
|
|
124
124
|
it "should pass the flag to the ES client" do
|
125
125
|
expect(::Elasticsearch::Client).to receive(:new) do |args|
|
126
|
-
expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :
|
126
|
+
expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :default)
|
127
127
|
end.and_return(es_client_double)
|
128
128
|
|
129
129
|
subject.register
|
@@ -200,7 +200,7 @@ describe "SSL options" do
|
|
200
200
|
:truststore => ssl_truststore_path,
|
201
201
|
:truststore_type => "jks",
|
202
202
|
:truststore_password => "foo",
|
203
|
-
:verify => :
|
203
|
+
:verify => :default,
|
204
204
|
:cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
|
205
205
|
:protocols => ["TLSv1.3"],
|
206
206
|
)
|
@@ -236,7 +236,7 @@ describe "SSL options" do
|
|
236
236
|
:ca_file => ssl_certificate_authorities_path,
|
237
237
|
:client_cert => ssl_certificate_path,
|
238
238
|
:client_key => ssl_key_path,
|
239
|
-
:verify => :
|
239
|
+
:verify => :default,
|
240
240
|
:cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
|
241
241
|
:protocols => ["TLSv1.3"],
|
242
242
|
)
|
@@ -4,7 +4,7 @@ require "logstash/plugin"
|
|
4
4
|
require "logstash/inputs/elasticsearch"
|
5
5
|
require_relative "../../../spec/es_helper"
|
6
6
|
|
7
|
-
describe LogStash::Inputs::Elasticsearch do
|
7
|
+
describe LogStash::Inputs::Elasticsearch, :integration => true do
|
8
8
|
|
9
9
|
SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
|
10
10
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.17.
|
4
|
+
version: 4.17.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-06-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -304,7 +304,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
304
304
|
- !ruby/object:Gem::Version
|
305
305
|
version: '0'
|
306
306
|
requirements: []
|
307
|
-
rubygems_version: 3.
|
307
|
+
rubygems_version: 3.2.33
|
308
308
|
signing_key:
|
309
309
|
specification_version: 4
|
310
310
|
summary: Reads query results from an Elasticsearch cluster
|