logstash-integration-jdbc 5.1.10 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f3116d5abaa413fd73af725208c422a1cf98961ce7095c4d88fd7f14cb6c59d
-  data.tar.gz: 0ef2f5fd309236b8ec0e14bb38d275e26b3d2002cdb172a2d30d53682606ddec
+  metadata.gz: 2119dbab43322bdc083cf11602e69e63bcb1a957b537f5f44cfcd2b611df1893
+  data.tar.gz: 651e016e43bb17db730cc95ad0fd44bed483b9edfa4bbe86e52ced95947bb4a4
 SHA512:
-  metadata.gz: d701317b4edbe221b2dd9a66d6e2591a0a78e8b9502373860656a5d95f552eee06ae752904687b58b2dfda274dc7c20e5da3e5bebbefbd98c538c107f49bec7a
-  data.tar.gz: 22a9383f113f01025ba2f694cd61845aa4ed08474efc70cb37694993babb9ec7d6ac3d95291bbf2fe5b22835a8923ab1802061d861da2e59d5b0e3041f7b18ed
+  metadata.gz: 0a90d6aa88365c9ddfe60bba20f6f06633733af4b910b75becc31c37a9e1a224d3aae8d5e62f03f8ae2a3ffd8a24d1b0635307aa152a836c18435a4b36ca1d5d
+  data.tar.gz: f6a6648d071bff9069f54448cfed03327cb0f5bb8c036d8b0c6e5adb949c69a75f923541fe6a1237eeb3e6fe35611ad0afd4f99e356e8dd42b1e2b19534bc5b7

data/CHANGELOG.md

@@ -1,7 +1,22 @@
+## 5.2.3
+  - Performance: avoid contention on scheduler execution [#103](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/103)
+
+## 5.2.2
+  - Feat: name scheduler threads + redirect error logging [#102](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/102)
+
+## 5.2.1
+  - Refactor: isolate paginated normal statement algorithm in a separate handler [#101](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/101)
+
+## 5.2.0
+  - Added `jdbc_paging_mode` option to choose if use `explicit` pagination in statements and avoid the initial count 
+    query or use `auto` to delegate to the underlying library [#95](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/95)
+
 ## 5.1.10
   - Refactor: to explicit Java (driver) class name loading [#96](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/96),
     the change is expected to provide a more robust fix for the driver loading issue [#83](https://github.com/logstash-plugins/logstash-integration-jdbc/issues/83).
 
+    NOTE: a fatal driver error will no longer keep reloading the pipeline and now leads to a system exit. 
+
   - Fix: regression due returning the Java driver class [#98](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/98)
 
 ## 5.1.9 (yanked)

data/docs/input-jdbc.asciidoc

@@ -129,6 +129,9 @@ Here is the list:
 |sql_last_value | The value used to calculate which rows to query. Before any query is run,
 this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and
 `tracking_column` is set. It is updated accordingly after subsequent queries are run.
+|offset, size| Values used with manual paging mode to explicitly implement the paging.
+Supported only if <<plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and
+<<plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value.
 |==========================================================
 
 Example:
@@ -153,7 +156,7 @@ NOTE: Not all JDBC accessible technologies will support prepared statements.
 With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on.
 Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array.
 
-NOTE: Building count queries around a prepared statement is not supported at this time and because jdbc paging uses count queries under the hood, jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements.
+NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements.
 
 Example:
 [source,ruby]
@@ -193,6 +196,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-jdbc_fetch_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-jdbc_page_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-jdbc_paging_enabled>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-jdbc_paging_mode>> |<<string,string>>, one of `["auto", "explicit"]`|No
 | <<plugins-{type}s-{plugin}-jdbc_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-jdbc_pool_timeout>> |<<number,number>>|No
@@ -373,6 +377,52 @@ result-set. The limit size is set with `jdbc_page_size`.
 
 Be aware that ordering is not guaranteed between queries.
 
+[id="plugins-{type}s-{plugin}-jdbc_paging_mode"]
+===== `jdbc_paging_mode`
+
+  * Value can be any of: `auto`, `explicit`
+  * Default value is `"auto"`
+
+Whether to use `explicit` or `auto` mode during the JDBC paging
+
+If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.).
+
+If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved.
+You have to write your own paging conditions in your statement configuration.
+The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query).
+When the number of rows returned by the query is not equal to `size`, SQL paging will be ended.
+Example:
+
+[source, ruby]
+------------------------------------------------------
+input {
+  jdbc {
+    statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset",
+    jdbc_paging_enabled => true,
+    jdbc_paging_mode => "explicit",
+    jdbc_page_size => 100000
+  }
+}
+------------------------------------------------------
+
+[source, ruby]
+------------------------------------------------------
+input {
+  jdbc {
+    statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)",
+    jdbc_paging_enabled => true,
+    jdbc_paging_mode => "explicit",
+    jdbc_page_size => 100000
+  }
+}
+------------------------------------------------------
+
+This mode can be considered in the following situations:
+
+. Performance issues encountered in default paging mode.
+. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want.
+. Your statement is a stored procedure, and the actual paging statement is inside it.
+
 [id="plugins-{type}s-{plugin}-jdbc_password"]
 ===== `jdbc_password`
 

data/lib/logstash/inputs/jdbc.rb

@@ -3,6 +3,7 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/plugin_mixins/jdbc/common"
 require "logstash/plugin_mixins/jdbc/jdbc"
+require "logstash/plugin_mixins/jdbc/scheduler"
 require "logstash/plugin_mixins/ecs_compatibility_support"
 require "logstash/plugin_mixins/ecs_compatibility_support/target_check"
 require "logstash/plugin_mixins/validator_support/field_reference_validation_adapter"
@@ -293,8 +294,17 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   def run(queue)
     load_driver
     if @schedule
-      @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
-      @scheduler.cron @schedule do
+      # input thread (Java) name example "[my-oracle]<jdbc"
+      @scheduler = LogStash::PluginMixins::Jdbc::Scheduler.new(
+          :max_work_threads => 1,
+          :thread_name => "[#{id}]<jdbc__scheduler",
+          # amount the scheduler thread sleeps between checking whether jobs
+          # should trigger, default is 0.3 which is a bit too often ...
+          # in theory the cron expression '* * * * * *' supports running jobs
+          # every second but this is very rare, we could potentially go higher
+          :frequency => 1.0,
+      )
+      @scheduler.schedule_cron @schedule do
         execute_query(queue)
       end
 

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -55,6 +55,9 @@ module LogStash  module PluginMixins module Jdbc
       # Be aware that ordering is not guaranteed between queries.
       config :jdbc_paging_enabled, :validate => :boolean, :default => false
 
+      # Which pagination mode to use, automatic pagination or explicitly defined in the query.
+      config :jdbc_paging_mode, :validate => [ "auto", "explicit" ], :default => "auto"
+
       # JDBC page size
       config :jdbc_page_size, :validate => :number, :default => 100000
 
@@ -211,13 +214,14 @@ module LogStash  module PluginMixins module Jdbc
         open_jdbc_connection
         sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
         @tracking_column_warning_sent = false
-        @statement_handler.perform_query(@database, @value_tracker.value, @jdbc_paging_enabled, @jdbc_page_size) do |row|
+        @statement_handler.perform_query(@database, @value_tracker.value) do |row|
           sql_last_value = get_column_value(row) if @use_column_value
           yield extract_values_from(row)
         end
         success = true
       rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError, Java::JavaSql::SQLException => e
-        details = { :exception => e.message }
+        details = { exception: e.class, message: e.message }
+        details[:cause] = e.cause.inspect if e.cause
         details[:backtrace] = e.backtrace if @logger.debug?
         @logger.warn("Exception when executing JDBC query", details)
       else

data/lib/logstash/plugin_mixins/jdbc/scheduler.rb

@@ -0,0 +1,147 @@
+require 'rufus/scheduler'
+
+require 'logstash/util/loggable'
+
+module LogStash module PluginMixins module Jdbc
+  class Scheduler < Rufus::Scheduler
+
+    include LogStash::Util::Loggable
+
+    # Rufus::Scheduler >= 3.4 moved the Time impl into a gem EoTime = ::EtOrbi::EoTime`
+    # Rufus::Scheduler 3.1 - 3.3 using it's own Time impl `Rufus::Scheduler::ZoTime`
+    TimeImpl = defined?(Rufus::Scheduler::EoTime) ? Rufus::Scheduler::EoTime :
+                   (defined?(Rufus::Scheduler::ZoTime) ? Rufus::Scheduler::ZoTime : ::Time)
+
+    # @overload
+    def timeout_jobs
+      # Rufus relies on `Thread.list` which is a blocking operation and with many schedulers
+      # (and threads) within LS will have a negative impact on performance as scheduler
+      # threads will end up waiting to obtain the `Thread.list` lock.
+      #
+      # However, this isn't necessary we can easily detect whether there are any jobs
+      # that might need to timeout: only when `@opts[:timeout]` is set causes worker thread(s)
+      # to have a `Thread.current[:rufus_scheduler_timeout]` that is not nil
+      return unless @opts[:timeout]
+      super
+    end
+
+    # @overload
+    def work_threads(query = :all)
+      if query == :__all_no_cache__ # special case from JobDecorator#start_work_thread
+        @_work_threads = nil # when a new worker thread is being added reset
+        return super(:all)
+      end
+
+      # Gets executed every time a job is triggered, we're going to cache the
+      # worker threads for this scheduler (to avoid `Thread.list`) - they only
+      # change when a new thread is being started from #start_work_thread ...
+      work_threads = @_work_threads
+      if work_threads.nil?
+        work_threads = threads.select { |t| t[:rufus_scheduler_work_thread] }
+        @_work_threads = work_threads
+      end
+
+      case query
+      when :active then work_threads.select { |t| t[:rufus_scheduler_job] }
+      when :vacant then work_threads.reject { |t| t[:rufus_scheduler_job] }
+      else work_threads
+      end
+    end
+
+    # @overload
+    def on_error(job, err)
+      details = { exception: err.class, message: err.message, backtrace: err.backtrace }
+      details[:cause] = err.cause if err.cause
+
+      details[:now] = debug_format_time(TimeImpl.now)
+      details[:last_time] = (debug_format_time(job.last_time) rescue nil)
+      details[:next_time] = (debug_format_time(job.next_time) rescue nil)
+      details[:job] = job
+
+      details[:opts] = @opts
+      details[:started_at] = started_at
+      details[:thread] = thread.inspect
+      details[:jobs_size] = @jobs.size
+      details[:work_threads_size] = work_threads.size
+      details[:work_queue_size] = work_queue.size
+
+      logger.error("Scheduler intercepted an error:", details)
+
+    rescue => e
+      logger.error("Scheduler failed in #on_error #{e.inspect}")
+    end
+
+    def debug_format_time(time)
+      # EtOrbi::EoTime used by (newer) Rufus::Scheduler has to_debug_s https://git.io/JyiPj
+      time.respond_to?(:to_debug_s) ? time.to_debug_s : time.strftime("%Y-%m-%dT%H:%M:%S.%L")
+    end
+    private :debug_format_time
+
+    # @private helper used by JobDecorator
+    def work_thread_name_prefix
+      ( @opts[:thread_name] || "#{@thread_key}_scheduler" ) + '_worker-'
+    end
+
+    protected
+
+    # @overload
+    def start
+      ret = super() # @thread[:name] = @opts[:thread_name] || "#{@thread_key}_scheduler"
+
+      # at least set thread.name for easier thread dump analysis
+      if @thread.is_a?(Thread) && @thread.respond_to?(:name=)
+        @thread.name = @thread[:name] if @thread[:name]
+      end
+
+      ret
+    end
+
+    # @overload
+    def do_schedule(job_type, t, callable, opts, return_job_instance, block)
+      job_or_id = super
+
+      job_or_id.extend JobDecorator if return_job_instance
+
+      job_or_id
+    end
+
+    module JobDecorator
+
+      def start_work_thread
+        prev_thread_count = @scheduler.work_threads.size
+
+        ret = super() # does not return Thread instance in 3.0
+
+        work_threads = @scheduler.work_threads(:__all_no_cache__)
+        while prev_thread_count == work_threads.size # very unlikely
+          Thread.pass
+          work_threads = @scheduler.work_threads(:__all_no_cache__)
+        end
+
+        work_thread_name_prefix = @scheduler.work_thread_name_prefix
+
+        work_threads.sort! do |t1, t2|
+          if t1[:name].nil?
+            t2[:name].nil? ? 0 : +1 # nils at the end
+          elsif t2[:name].nil?
+            t1[:name].nil? ? 0 : -1
+          else
+            t1[:name] <=> t2[:name]
+          end
+        end
+
+        work_threads.each_with_index do |thread, i|
+          unless thread[:name]
+            thread[:name] = "#{work_thread_name_prefix}#{sprintf('%02i', i)}"
+            thread.name = thread[:name] if thread.respond_to?(:name=)
+            # e.g. "[oracle]<jdbc_scheduler_worker-00"
+          end
+        end
+
+        ret
+      end
+
+    end
+
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb

@@ -3,7 +3,19 @@
 module LogStash module PluginMixins module Jdbc
   class StatementHandler
     def self.build_statement_handler(plugin, logger)
-      klass = plugin.use_prepared_statements ? PreparedStatementHandler : NormalStatementHandler
+      if plugin.use_prepared_statements
+        klass = PreparedStatementHandler
+      else
+        if plugin.jdbc_paging_enabled
+          if plugin.jdbc_paging_mode == "explicit"
+            klass = ExplicitPagingModeStatementHandler
+          else
+            klass = PagedNormalStatementHandler
+          end
+        else
+          klass = NormalStatementHandler
+        end
+      end
       klass.new(plugin, logger)
     end
 
@@ -25,22 +37,14 @@ module LogStash module PluginMixins module Jdbc
   end
 
   class NormalStatementHandler < StatementHandler
-    # Performs the query, respecting our pagination settings, yielding once per row of data
+    # Performs the query, yielding once per row of data
     # @param db [Sequel::Database]
-    # @param sql_last_value [Integet|DateTime|Time]
+    # @param sql_last_value [Integer|DateTime|Time]
     # @yieldparam row [Hash{Symbol=>Object}]
-    def perform_query(db, sql_last_value, jdbc_paging_enabled, jdbc_page_size)
+    def perform_query(db, sql_last_value)
       query = build_query(db, sql_last_value)
-      if jdbc_paging_enabled
-        query.each_page(jdbc_page_size) do |paged_dataset|
-          paged_dataset.each do |row|
-            yield row
-          end
-        end
-      else
-        query.each do |row|
-          yield row
-        end
+      query.each do |row|
+        yield row
       end
     end
 
@@ -67,6 +71,48 @@ module LogStash module PluginMixins module Jdbc
     end
   end
 
+  class PagedNormalStatementHandler < NormalStatementHandler
+    attr_reader :jdbc_page_size
+
+    # Performs the query, respecting our pagination settings, yielding once per row of data
+    # @param db [Sequel::Database]
+    # @param sql_last_value [Integer|DateTime|Time]
+    # @yieldparam row [Hash{Symbol=>Object}]
+    def perform_query(db, sql_last_value)
+      query = build_query(db, sql_last_value)
+      query.each_page(@jdbc_page_size) do |paged_dataset|
+        paged_dataset.each do |row|
+          yield row
+        end
+      end
+    end
+
+    def post_init(plugin)
+      super(plugin)
+      @jdbc_page_size = plugin.jdbc_page_size
+    end
+  end
+
+  class ExplicitPagingModeStatementHandler < PagedNormalStatementHandler
+    # Performs the query, respecting our pagination settings, yielding once per row of data
+    # @param db [Sequel::Database]
+    # @param sql_last_value [Integer|DateTime|Time]
+    # @yieldparam row [Hash{Symbol=>Object}]
+    def perform_query(db, sql_last_value)
+      query = build_query(db, sql_last_value)
+      offset = 0
+      loop do
+        rows_in_page = 0
+        query.with_sql(query.sql, offset: offset, size: jdbc_page_size).each do |row|
+          yield row
+          rows_in_page += 1
+        end
+        break unless rows_in_page == jdbc_page_size
+        offset += jdbc_page_size
+      end
+    end
+  end
+
   class PreparedStatementHandler < StatementHandler
     attr_reader :name, :bind_values_array, :statement_prepared, :prepared
 
@@ -74,7 +120,7 @@ module LogStash module PluginMixins module Jdbc
     # @param db [Sequel::Database]
     # @param sql_last_value [Integet|DateTime|Time]
     # @yieldparam row [Hash{Symbol=>Object}]
-    def perform_query(db, sql_last_value, jdbc_paging_enabled, jdbc_page_size)
+    def perform_query(db, sql_last_value)
       query = build_query(db, sql_last_value)
       query.each do |row|
         yield row

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.1.10'
+  s.version         = '5.2.3'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -34,8 +34,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
-  # 3.5 limitation is required for jdbc-static loading schedule
-  s.add_runtime_dependency 'rufus-scheduler', '< 3.5'
+  s.add_runtime_dependency 'rufus-scheduler', '~> 3.0.9'
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'

data/spec/inputs/integration/integ_spec.rb

@@ -70,7 +70,7 @@ describe LogStash::Inputs::Jdbc, :integration => true do
       plugin.register
       expect( plugin ).to receive(:log_java_exception)
       expect(plugin.logger).to receive(:warn).once.with("Exception when executing JDBC query",
-                                                        hash_including(:exception => instance_of(String)))
+                                                        hash_including(:message => instance_of(String)))
       q = Queue.new
       expect{ plugin.run(q) }.not_to raise_error
     end

data/spec/inputs/jdbc_spec.rb

@@ -329,6 +329,39 @@ describe LogStash::Inputs::Jdbc do
 
   end
 
+  context "when iterating result-set via explicit paging mode" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from test_table OFFSET :offset ROWS FETCH NEXT :size ROWS ONLY",
+        "jdbc_paging_enabled" => true,
+        "jdbc_paging_mode" => "explicit",
+        "jdbc_page_size" => 10
+      }
+    end
+
+    let(:num_rows) { 15 }
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should fetch all rows" do
+      num_rows.times do
+        db[:test_table].insert(:num => 1, :custom_time => Time.now.utc, :created_at => Time.now.utc)
+      end
+
+      plugin.run(queue)
+
+      expect(queue.size).to eq(num_rows)
+    end
+
+  end
+
   context "when using target option" do
     let(:settings) do
       {

data/spec/plugin_mixins/jdbc/scheduler_spec.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin_mixins/jdbc/scheduler"
+
+describe LogStash::PluginMixins::Jdbc::Scheduler do
+
+  let(:thread_name) { '[test]<jdbc_scheduler' }
+
+  let(:opts) do
+    { :max_work_threads => 2, :thread_name => thread_name }
+  end
+
+  subject(:scheduler) { LogStash::PluginMixins::Jdbc::Scheduler.new(opts) }
+
+  after { scheduler.stop(:wait) }
+
+  it "sets scheduler thread name" do
+    expect( scheduler.thread.name ).to include thread_name
+  end
+
+  context 'cron schedule' do
+
+    before do
+      scheduler.schedule_cron('* * * * * *') { sleep 1.25 } # every second
+    end
+
+    it "sets worker thread names" do
+      sleep 3.0
+      threads = scheduler.work_threads
+      threads.sort! { |t1, t2| (t1.name || '') <=> (t2.name || '') }
+
+      expect( threads.size ).to eql 2
+      expect( threads.first.name ).to eql "#{thread_name}_worker-00"
+      expect( threads.last.name ).to eql "#{thread_name}_worker-01"
+    end
+
+  end
+
+  context 'every 1s' do
+
+    before do
+      scheduler.schedule_in('1s') { raise 'TEST' } # every second
+    end
+
+    it "logs errors handled" do
+      expect( scheduler.logger ).to receive(:error).with /Scheduler intercepted an error/, hash_including(:message => 'TEST')
+      sleep 1.5
+    end
+
+  end
+
+  context 'work threads' do
+
+    let(:opts) { super().merge :max_work_threads => 3 }
+
+    let(:counter) { java.util.concurrent.atomic.AtomicLong.new(0) }
+
+    before do
+      scheduler.schedule_cron('* * * * * *') { counter.increment_and_get; sleep 3.25 } # every second
+    end
+
+    it "are working" do
+      sleep(0.05) while counter.get == 0
+      expect( scheduler.work_threads.size ).to eql 1
+      sleep(0.05) while counter.get == 1
+      expect( scheduler.work_threads.size ).to eql 2
+      sleep(0.05) while counter.get == 2
+      expect( scheduler.work_threads.size ).to eql 3
+
+      sleep 1.25
+      expect( scheduler.work_threads.size ).to eql 3
+      sleep 1.25
+      expect( scheduler.work_threads.size ).to eql 3
+    end
+
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.1.10
+  version: 5.2.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-16 00:00:00.000000000 Z
+date: 2022-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -131,17 +131,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: 3.0.9
   name: rufus-scheduler
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: 3.0.9
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -278,6 +278,7 @@ files:
 - lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb
 - lib/logstash/plugin_mixins/jdbc/common.rb
 - lib/logstash/plugin_mixins/jdbc/jdbc.rb
+- lib/logstash/plugin_mixins/jdbc/scheduler.rb
 - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
 - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
 - lib/logstash/plugin_mixins/jdbc_streaming.rb
@@ -306,6 +307,7 @@ files:
 - spec/helpers/derbyrun.jar
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
+- spec/plugin_mixins/jdbc/scheduler_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
 - vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
 - vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
@@ -358,4 +360,5 @@ test_files:
 - spec/helpers/derbyrun.jar
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
+- spec/plugin_mixins/jdbc/scheduler_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb