logstash-input-elasticsearch 4.17.0 → 4.17.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43e3c8e44ba8bd1ce4065a1410b26a63d621180d0354b4b4aa8f3891962d9986
-  data.tar.gz: c9f6d3b53d1bb90c9ad77eab86b92818128f310aa95a80e2908bdc5b981a780f
+  metadata.gz: 54b223c24702d4e9eb27efc05d351dbe9138d7be576a72e552bea3ef40254bb5
+  data.tar.gz: 697ade265dd83a9b87a6802e650a9100e64195315577e927c8228cdb6b18e720
 SHA512:
-  metadata.gz: 5799d6ef93349a2ca9a6408bba0bbaa78f5c985137a4d126c70efe53122b7eac4d1fcda48aeb51f629651ea04e4457e9bb9705b760744b0ff22cd462f617c7b8
-  data.tar.gz: f0057cebd50dc79156d8f05be48341f009366fc9057df0f61d50327a3cecf0a95ebccd05bed120173ab429caab295d5b71f25edc04232fd36ecf862ad58bd974
+  metadata.gz: 50ac2baa3e825cca7ffd51321bf42530b5c6f32dd88db94c9a0ce5ec2e978de5e0e82f645d6c8116972b40e6efae01782a73dc38bbda50d513df1bb88acf6e75
+  data.tar.gz: 50290a768d6f8985cef40b393bdf4d80842416825c5d3e8d835485b9a96adf4f52cbfd654fb687c92cc22fdcc616cf70147f0d9ded807e1a07d135f23e6a2fd7

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.17.2
+  - Fixes a regression introduced in 4.17.0 which could prevent a connection from being established to Elasticsearch in some SSL configurations [#193](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/193)
+
+## 4.17.1
+  - Fix: scroll slice high memory consumption [#189](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/189)
+
 ## 4.17.0
   - Added SSL settings for: [#185](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/185)
     - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme

data/docs/index.asciidoc

@@ -353,6 +353,7 @@ documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a l
 The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message.
 The default is 0 (no retry). This value should be equal to or greater than zero.
 
+NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible.
 
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 

data/lib/logstash/inputs/elasticsearch.rb

@@ -256,6 +256,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
+  attr_reader :pipeline_id
+
   def initialize(params={})
     super(params)
 
@@ -267,6 +269,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def register
     require "rufus/scheduler"
 
+    @pipeline_id = execution_context&.pipeline_id || 'main'
+
     fill_hosts_from_cloud_id
     setup_ssl_params!
 
@@ -326,20 +330,22 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def do_run(output_queue)
     # if configured to run a single slice, don't bother spinning up threads
     if @slices.nil? || @slices <= 1
-      success, events = retryable_slice
-      success && events.each { |event| output_queue << event }
-      return
+      return retryable(JOB_NAME) do
+        do_run_slice(output_queue)
+      end
     end
 
     logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
 
-    slice_results = parallel_slice # array of tuple(ok, events)
 
-    # insert events to queue if all slices success
-    if slice_results.all?(&:first)
-      slice_results.flat_map { |success, events| events }
-                  .each { |event| output_queue << event }
-    end
+    @slices.times.map do |slice_id|
+      Thread.new do
+        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
+        retryable(JOB_NAME) do
+          do_run_slice(output_queue, slice_id)
+        end
+      end
+    end.map(&:join)
 
     logger.trace("#{@slices} slices completed")
   end
@@ -347,42 +353,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def retryable(job_name, &block)
     begin
       stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
-      output = stud_try.try((@retries + 1).times) { yield }
-      [true, output]
+      stud_try.try((@retries + 1).times) { yield }
     rescue => e
       error_details = {:message => e.message, :cause => e.cause}
       error_details[:backtrace] = e.backtrace if logger.debug?
       logger.error("Tried #{job_name} unsuccessfully", error_details)
-      [false, nil]
     end
   end
 
-
-  # @return [(ok, events)] : Array of tuple(Boolean, [Logstash::Event])
-  def parallel_slice
-    pipeline_id = execution_context&.pipeline_id || 'main'
-    @slices.times.map do |slice_id|
-      Thread.new do
-        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
-        retryable_slice(slice_id)
-      end
-    end.map do |t|
-      t.join
-      t.value
-    end
-  end
-
-  # @param scroll_id [Integer]
-  # @return (ok, events) [Boolean, Array(Logstash::Event)]
-  def retryable_slice(slice_id=nil)
-    retryable(JOB_NAME) do
-      output = []
-      do_run_slice(output, slice_id)
-      output
-    end
-  end
-
-
   def do_run_slice(output_queue, slice_id=nil)
     slice_query = @base_query
     slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
@@ -543,7 +521,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
                         "to make sure your data is secure set `ssl_verification_mode => full`"
           ssl_options[:verify] = :disable
         else
-          ssl_options[:verify] = :strict
+          # Manticore's :default maps to Apache HTTP Client's DefaultHostnameVerifier,
+          # which is the modern STRICT verifier that replaces the deprecated StrictHostnameVerifier
+          ssl_options[:verify] = :default
       end
     end
 

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.17.0'
+  s.version         = '4.17.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -204,7 +204,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'with `slices => 1`' do
       let(:slices) { 1 }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -215,7 +215,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'without slices directive' do
       let(:config) { super().tap { |h| h.delete('slices') } }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -414,18 +414,19 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
           plugin.register
 
-          expect(client).to receive(:clear_scroll).and_return(nil)
+          expect(client).to receive(:clear_scroll).twice.and_return(nil)
 
-          # SLICE0 is a three-page scroll in which the second page throw exception
+          # SLICE0 is a three-page scroll
           slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
           expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
-          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_raise("boom")
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
           allow(client).to receive(:ping)
 
-          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          # SLICE1 is a two-page scroll in which the last page throws exception
           slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
           expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
-          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_raise("boom")
 
           synchronize_method!(plugin, :scroll_request)
           synchronize_method!(plugin, :search_request)
@@ -433,18 +434,22 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
         let(:client) { Elasticsearch::Client.new }
 
-        it 'does not insert event to queue' do
-          expect(plugin).to receive(:parallel_slice).and_wrap_original do |m, *args|
-            slice0, slice1 = m.call
-            expect(slice0[0]).to be_falsey
-            expect(slice1[0]).to be_truthy
-            expect(slice1[1].size).to eq(4) # four items from SLICE1
-            [slice0, slice1]
+        it 'insert event to queue without waiting other slices' do
+          expect(plugin).to receive(:do_run_slice).twice.and_wrap_original do |m, *args|
+            q = args[0]
+            slice_id = args[1]
+            if slice_id == 0
+              m.call(*args)
+              expect(q.size).to eq(3)
+            else
+              sleep(1)
+              m.call(*args)
+            end
           end
 
           queue = Queue.new
           plugin.run(queue)
-          expect(queue.size).to eq(0)
+          expect(queue.size).to eq(5)
         end
       end
     end

data/spec/inputs/elasticsearch_ssl_spec.rb

@@ -123,7 +123,7 @@ describe "SSL options" do
 
       it "should pass the flag to the ES client" do
         expect(::Elasticsearch::Client).to receive(:new) do |args|
-          expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :strict)
+          expect(args[:ssl]).to match hash_including(:ssl => true, :verify => :default)
         end.and_return(es_client_double)
 
         subject.register
@@ -200,7 +200,7 @@ describe "SSL options" do
                                       :truststore => ssl_truststore_path,
                                       :truststore_type => "jks",
                                       :truststore_password => "foo",
-                                      :verify => :strict,
+                                      :verify => :default,
                                       :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
                                       :protocols => ["TLSv1.3"],
                                     )
@@ -236,7 +236,7 @@ describe "SSL options" do
                                       :ca_file => ssl_certificate_authorities_path,
                                       :client_cert => ssl_certificate_path,
                                       :client_key => ssl_key_path,
-                                      :verify => :strict,
+                                      :verify => :default,
                                       :cipher_suites => ["TLS_DHE_RSA_WITH_AES_256_CBC_SHA256"],
                                       :protocols => ["TLSv1.3"],
                                     )

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -4,7 +4,7 @@ require "logstash/plugin"
 require "logstash/inputs/elasticsearch"
 require_relative "../../../spec/es_helper"
 
-describe LogStash::Inputs::Elasticsearch do
+describe LogStash::Inputs::Elasticsearch, :integration => true do
 
   SECURE_INTEGRATION = ENV['SECURE_INTEGRATION'].eql? 'true'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.17.0
+  version: 4.17.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-10 00:00:00.000000000 Z
+date: 2023-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -304,7 +304,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster