logstash-integration-jdbc 5.2.2 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f43be09f1996c0e4effd6914055fa643549925433ab4512f4e3a0ed1fa1933c
-  data.tar.gz: 346cb42a27eaaa3f51e7cf1ba3d52fb332f278dfe8cecf7d1aa200af42ba4f5b
+  metadata.gz: 2119dbab43322bdc083cf11602e69e63bcb1a957b537f5f44cfcd2b611df1893
+  data.tar.gz: 651e016e43bb17db730cc95ad0fd44bed483b9edfa4bbe86e52ced95947bb4a4
 SHA512:
-  metadata.gz: 3a75a4a4165e04178384a66740de3bce2981c2d019a8bd2192519a725bfe88d5ddf7469cf2a2aebda12b1b48b2117817171da2f459d283c3e3e16d86422c4c06
-  data.tar.gz: ae15ead4d5efd52cc67bf58ad1d994bde11443bbc0e083d56f695ddc7163be44454ee1db1a95b81ec884c51a790f2e7582733df4f5faf6b9d4715a0685a31c92
+  metadata.gz: 0a90d6aa88365c9ddfe60bba20f6f06633733af4b910b75becc31c37a9e1a224d3aae8d5e62f03f8ae2a3ffd8a24d1b0635307aa152a836c18435a4b36ca1d5d
+  data.tar.gz: f6a6648d071bff9069f54448cfed03327cb0f5bb8c036d8b0c6e5adb949c69a75f923541fe6a1237eeb3e6fe35611ad0afd4f99e356e8dd42b1e2b19534bc5b7

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.2.3
+  - Performance: avoid contention on scheduler execution [#103](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/103)
+
 ## 5.2.2
   - Feat: name scheduler threads + redirect error logging [#102](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/102)
 

data/lib/logstash/inputs/jdbc.rb

@@ -296,7 +296,13 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
     if @schedule
       # input thread (Java) name example "[my-oracle]<jdbc"
       @scheduler = LogStash::PluginMixins::Jdbc::Scheduler.new(
-          :max_work_threads => 1, :thread_name => "[#{id}]<jdbc__scheduler"
+          :max_work_threads => 1,
+          :thread_name => "[#{id}]<jdbc__scheduler",
+          # amount the scheduler thread sleeps between checking whether jobs
+          # should trigger, default is 0.3 which is a bit too often ...
+          # in theory the cron expression '* * * * * *' supports running jobs
+          # every second but this is very rare, we could potentially go higher
+          :frequency => 1.0,
       )
       @scheduler.schedule_cron @schedule do
         execute_query(queue)

data/lib/logstash/plugin_mixins/jdbc/scheduler.rb

@@ -12,6 +12,42 @@ module LogStash module PluginMixins module Jdbc
     TimeImpl = defined?(Rufus::Scheduler::EoTime) ? Rufus::Scheduler::EoTime :
                    (defined?(Rufus::Scheduler::ZoTime) ? Rufus::Scheduler::ZoTime : ::Time)
 
+    # @overload
+    def timeout_jobs
+      # Rufus relies on `Thread.list` which is a blocking operation and with many schedulers
+      # (and threads) within LS will have a negative impact on performance as scheduler
+      # threads will end up waiting to obtain the `Thread.list` lock.
+      #
+      # However, this isn't necessary we can easily detect whether there are any jobs
+      # that might need to timeout: only when `@opts[:timeout]` is set causes worker thread(s)
+      # to have a `Thread.current[:rufus_scheduler_timeout]` that is not nil
+      return unless @opts[:timeout]
+      super
+    end
+
+    # @overload
+    def work_threads(query = :all)
+      if query == :__all_no_cache__ # special case from JobDecorator#start_work_thread
+        @_work_threads = nil # when a new worker thread is being added reset
+        return super(:all)
+      end
+
+      # Gets executed every time a job is triggered, we're going to cache the
+      # worker threads for this scheduler (to avoid `Thread.list`) - they only
+      # change when a new thread is being started from #start_work_thread ...
+      work_threads = @_work_threads
+      if work_threads.nil?
+        work_threads = threads.select { |t| t[:rufus_scheduler_work_thread] }
+        @_work_threads = work_threads
+      end
+
+      case query
+      when :active then work_threads.select { |t| t[:rufus_scheduler_job] }
+      when :vacant then work_threads.reject { |t| t[:rufus_scheduler_job] }
+      else work_threads
+      end
+    end
+
     # @overload
     def on_error(job, err)
       details = { exception: err.class, message: err.message, backtrace: err.backtrace }
@@ -76,10 +112,10 @@ module LogStash module PluginMixins module Jdbc
 
         ret = super() # does not return Thread instance in 3.0
 
-        work_threads = @scheduler.work_threads
+        work_threads = @scheduler.work_threads(:__all_no_cache__)
         while prev_thread_count == work_threads.size # very unlikely
           Thread.pass
-          work_threads = @scheduler.work_threads
+          work_threads = @scheduler.work_threads(:__all_no_cache__)
         end
 
         work_thread_name_prefix = @scheduler.work_thread_name_prefix

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.2.2'
+  s.version         = '5.2.3'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/plugin_mixins/jdbc/scheduler_spec.rb

@@ -49,4 +49,30 @@ describe LogStash::PluginMixins::Jdbc::Scheduler do
 
   end
 
+  context 'work threads' do
+
+    let(:opts) { super().merge :max_work_threads => 3 }
+
+    let(:counter) { java.util.concurrent.atomic.AtomicLong.new(0) }
+
+    before do
+      scheduler.schedule_cron('* * * * * *') { counter.increment_and_get; sleep 3.25 } # every second
+    end
+
+    it "are working" do
+      sleep(0.05) while counter.get == 0
+      expect( scheduler.work_threads.size ).to eql 1
+      sleep(0.05) while counter.get == 1
+      expect( scheduler.work_threads.size ).to eql 2
+      sleep(0.05) while counter.get == 2
+      expect( scheduler.work_threads.size ).to eql 3
+
+      sleep 1.25
+      expect( scheduler.work_threads.size ).to eql 3
+      sleep 1.25
+      expect( scheduler.work_threads.size ).to eql 3
+    end
+
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.2.2
+  version: 5.2.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-19 00:00:00.000000000 Z
+date: 2022-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement