logstash-input-elasticsearch 4.17.0 → 4.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43e3c8e44ba8bd1ce4065a1410b26a63d621180d0354b4b4aa8f3891962d9986
-  data.tar.gz: c9f6d3b53d1bb90c9ad77eab86b92818128f310aa95a80e2908bdc5b981a780f
+  metadata.gz: 9cdbda91c48e11353b49426f1d9c01e461698e507506563848045560d05421d0
+  data.tar.gz: 367351221d9028775410a4ef272de5bf6bad66ab82d91aedacb91ef462c5ff56
 SHA512:
-  metadata.gz: 5799d6ef93349a2ca9a6408bba0bbaa78f5c985137a4d126c70efe53122b7eac4d1fcda48aeb51f629651ea04e4457e9bb9705b760744b0ff22cd462f617c7b8
-  data.tar.gz: f0057cebd50dc79156d8f05be48341f009366fc9057df0f61d50327a3cecf0a95ebccd05bed120173ab429caab295d5b71f25edc04232fd36ecf862ad58bd974
+  metadata.gz: 88071a99d4ee00c356c2729cdb0d873e7b0ecb067f02e3ccc76c83ec86166d4b4c1daa036d6943a5913339b524e69ecf3f741d17cd871cdba8cf19f59a371016
+  data.tar.gz: 1b20527dd5f3f3ca00bed248b6cdb57dc81d816f56d3f3d08c393fec4fdcf37f57f8ba61dec957efe03cf6f8695426ef544780e32d7b6818671ae0f0d6930b90

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.17.1
+  - Fix: scroll slice high memory consumption [#189](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/189)
+
 ## 4.17.0
   - Added SSL settings for: [#185](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/185)
     - `ssl_enabled`: Enable/disable the SSL settings. If not provided, the value is inferred from the hosts scheme

data/docs/index.asciidoc

@@ -353,6 +353,7 @@ documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a l
 The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message.
 The default is 0 (no retry). This value should be equal to or greater than zero.
 
+NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible.
 
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 

data/lib/logstash/inputs/elasticsearch.rb

@@ -256,6 +256,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
+  attr_reader :pipeline_id
+
   def initialize(params={})
     super(params)
 
@@ -267,6 +269,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def register
     require "rufus/scheduler"
 
+    @pipeline_id = execution_context&.pipeline_id || 'main'
+
     fill_hosts_from_cloud_id
     setup_ssl_params!
 
@@ -326,20 +330,22 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def do_run(output_queue)
     # if configured to run a single slice, don't bother spinning up threads
     if @slices.nil? || @slices <= 1
-      success, events = retryable_slice
-      success && events.each { |event| output_queue << event }
-      return
+      return retryable(JOB_NAME) do
+        do_run_slice(output_queue)
+      end
     end
 
     logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
 
-    slice_results = parallel_slice # array of tuple(ok, events)
 
-    # insert events to queue if all slices success
-    if slice_results.all?(&:first)
-      slice_results.flat_map { |success, events| events }
-                  .each { |event| output_queue << event }
-    end
+    @slices.times.map do |slice_id|
+      Thread.new do
+        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
+        retryable(JOB_NAME) do
+          do_run_slice(output_queue, slice_id)
+        end
+      end
+    end.map(&:join)
 
     logger.trace("#{@slices} slices completed")
   end
@@ -347,42 +353,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def retryable(job_name, &block)
     begin
       stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
-      output = stud_try.try((@retries + 1).times) { yield }
-      [true, output]
+      stud_try.try((@retries + 1).times) { yield }
     rescue => e
       error_details = {:message => e.message, :cause => e.cause}
       error_details[:backtrace] = e.backtrace if logger.debug?
       logger.error("Tried #{job_name} unsuccessfully", error_details)
-      [false, nil]
     end
   end
 
-
-  # @return [(ok, events)] : Array of tuple(Boolean, [Logstash::Event])
-  def parallel_slice
-    pipeline_id = execution_context&.pipeline_id || 'main'
-    @slices.times.map do |slice_id|
-      Thread.new do
-        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
-        retryable_slice(slice_id)
-      end
-    end.map do |t|
-      t.join
-      t.value
-    end
-  end
-
-  # @param scroll_id [Integer]
-  # @return (ok, events) [Boolean, Array(Logstash::Event)]
-  def retryable_slice(slice_id=nil)
-    retryable(JOB_NAME) do
-      output = []
-      do_run_slice(output, slice_id)
-      output
-    end
-  end
-
-
   def do_run_slice(output_queue, slice_id=nil)
     slice_query = @base_query
     slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.17.0'
+  s.version         = '4.17.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -204,7 +204,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'with `slices => 1`' do
       let(:slices) { 1 }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -215,7 +215,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'without slices directive' do
       let(:config) { super().tap { |h| h.delete('slices') } }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -414,18 +414,19 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
           plugin.register
 
-          expect(client).to receive(:clear_scroll).and_return(nil)
+          expect(client).to receive(:clear_scroll).twice.and_return(nil)
 
-          # SLICE0 is a three-page scroll in which the second page throw exception
+          # SLICE0 is a three-page scroll
           slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
           expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
-          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_raise("boom")
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
           allow(client).to receive(:ping)
 
-          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          # SLICE1 is a two-page scroll in which the last page throws exception
           slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
           expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
-          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_raise("boom")
 
           synchronize_method!(plugin, :scroll_request)
           synchronize_method!(plugin, :search_request)
@@ -433,18 +434,22 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
         let(:client) { Elasticsearch::Client.new }
 
-        it 'does not insert event to queue' do
-          expect(plugin).to receive(:parallel_slice).and_wrap_original do |m, *args|
-            slice0, slice1 = m.call
-            expect(slice0[0]).to be_falsey
-            expect(slice1[0]).to be_truthy
-            expect(slice1[1].size).to eq(4) # four items from SLICE1
-            [slice0, slice1]
+        it 'insert event to queue without waiting other slices' do
+          expect(plugin).to receive(:do_run_slice).twice.and_wrap_original do |m, *args|
+            q = args[0]
+            slice_id = args[1]
+            if slice_id == 0
+              m.call(*args)
+              expect(q.size).to eq(3)
+            else
+              sleep(1)
+              m.call(*args)
+            end
           end
 
           queue = Queue.new
           plugin.run(queue)
-          expect(queue.size).to eq(0)
+          expect(queue.size).to eq(5)
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.17.0
+  version: 4.17.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-10 00:00:00.000000000 Z
+date: 2023-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement