logstash-integration-jdbc 5.5.3 → 5.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a28e2a3a37342bc631ed300da7fa687e99fc879d8cc599669d64e2cfc6418177
-  data.tar.gz: 38b675462f29af81148dbdcc58144a2e3df932b6c6a44825b580a3468ac90acb
+  metadata.gz: b8af04cbdc4227939fa52c842815d13ffe85864f4ae0b6e9eade6f393edce935
+  data.tar.gz: b699ebedee5e9f982605998031e00933a69adac1b10018077e0411bc0dcae82c
 SHA512:
-  metadata.gz: b3af9d1d0c7ddc88b8102d7fafb96a9c379a734266698a3136e9e9522f640f64f4f019ea5820c9bc382a460df5f47f2fc62bb6cf8c3ad47ad84f5069710831cd
-  data.tar.gz: 41a6e399b350068d1ed337c158b94f9457f0ace2f31588888db8c99b9e20a39f7fe0f06ab4e809cd01c7a635c9ede2a00629aa1ecf04b9cd86951fdda5c9a828
+  metadata.gz: b25765b2945b4f6ad118a995ddc2a2350bca8a5390421f0707258f0cc65929cc5f06ce2bb317665338070e88a93c2442eeaf3604ca3325dfc3c511cecc58479f
+  data.tar.gz: 6ece1fa9e292c209d884e307db110ac64bca886de473dff145420c2a786174f6a87c66af19a8154c893f48dcecbd142d09af2918cee913bd483baa4813e55127

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.6.1
+  - Fixes an issue where the `jdbc_static` filter's throughput was artificially limited to 4 concurrent queries, causing the plugin to become a bottleneck in pipelines with more than 4 workers. Each instance of the plugin is now limited to 16 concurrent queries, with increased timeouts to eliminate enrichment failures [#187](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/187)
+
+## 5.6.0
+  - Support other rufus scheduling options in JDBC Input [#183](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/183)
+
 ## 5.5.3
   - [DOC] Rework inline comment to a callout in preparation for upcoming MD conversion [#181](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/181)
 

data/docs/input-jdbc.asciidoc

@@ -41,9 +41,9 @@ options for more info.
 
 Input from this plugin can be scheduled to run periodically according to a specific
 schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler].
-The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ).
+The syntax is either cron-like with some extensions specific to Rufus (e.g. timezone support ) if using the `schedule` option or periodic when using `period` or `interval` option.
 
-Examples:
+Examples for `schedule`:
 
 |==========================================================
 | `* 5 * 1-3 *`               | will execute every minute of 5am every day of January through March.
@@ -51,9 +51,21 @@ Examples:
 | `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day.
 |==========================================================
 
+Examples for `period` or `interval`:
+
+|==========================================================
+| `1m`                        | will execute every minute
+| `3h10m`                     | will execute every three hours and 10 minutes
+|==========================================================
 
 Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
 
+`interval` jobs trigger, execute and then trigger again after the interval elapsed.
+
+`period` jobs try to trigger following the frequency they were scheduled with.
+
+You can only use one of `interval`, `period` or `schedule` at the same time.
+
 ==== State
 
 The plugin will persist the `sql_last_value` parameter in the form of a
@@ -212,6 +224,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-columns_charset>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-connection_retry_attempts>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-connection_retry_attempts_wait_time>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-interval>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-jdbc_connection_string>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-jdbc_default_timezone>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-jdbc_driver_class>> |<<string,string>>|Yes
@@ -229,6 +242,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-last_run_metadata_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-lowercase_column_names>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-period>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-plugin_timezone>> |<<string,string>>, one of `["local", "utc"]`|No
 | <<plugins-{type}s-{plugin}-prepared_statement_bind_values>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-prepared_statement_name>> |<<string,string>>|No
@@ -298,6 +312,16 @@ Maximum number of times to try connecting to database
 
 Number of seconds to sleep between connection attempts
 
+[id="plugins-{type}s-{plugin}-interval"]
+===== `interval`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This takes a string in the form of `1h`, `1m`, to denote a time interval. `interval` jobs trigger, execute and trigger again after the provided time interval has elapsed.
+
+There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once.
+
 [id="plugins-{type}s-{plugin}-jdbc_connection_string"]
 ===== `jdbc_connection_string`
 
@@ -538,6 +562,16 @@ Whether to force the lowercasing of identifier fields
 
 Hash of query parameter, for example `{ "target_id" => "321" }`
 
+[id="plugins-{type}s-{plugin}-period"]
+===== `period`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+This takes a string in the form of `1h`, `1m`, to denote a time interval. `period` jobs try hard to trigger following the frequency they were scheduled with.
+
+There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once.
+
 [id="plugins-{type}s-{plugin}-prepared_statement_bind_values"]
 ===== `prepared_statement_bind_values`
 
@@ -568,11 +602,9 @@ Whether to save state or not in <<plugins-{type}s-{plugin}-last_run_metadata_pat
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-Schedule of when to periodically run statement, in Cron format
-for example: "* * * * *" (execute query every minute, on the minute)
+Schedule of when to periodically run statement, in Cron format for example: "* * * * *" (execute query every minute, on the minute)
 
-There is no schedule by default. If no schedule is given, then the statement is run
-exactly once.
+There is no schedule by default. If no scheduling statement is given, then the statement is run exactly once.
 
 [id="plugins-{type}s-{plugin}-sequel_opts"]
 ===== `sequel_opts`

data/lib/logstash/filters/jdbc/read_write_database.rb

@@ -103,6 +103,11 @@ module LogStash module Filters module Jdbc
       super
       # get a fair reentrant read write lock
       @rwlock = java.util.concurrent.locks.ReentrantReadWriteLock.new(true)
+
+      # configure the connection pool to reduce the chances of
+      # a worker thread being unable to acquire a connection.
+      @options_hash[:max_connections] = 16 # sequel default: 4
+      @options_hash[:pool_timeout] = 30    # sequel default: 5
     end
   end
 end end end

data/lib/logstash/inputs/jdbc.rb

@@ -180,6 +180,20 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   # exactly once.
   config :schedule, :validate => :string
 
+  # Interval of how soon to run statement again after completion
+  # for example: "1m" (execute again 1 minute after completion)
+  #
+  # There is no interval by default. If no interval is given, then the statement is run
+  # exactly once.
+  config :interval, :validate => :string
+
+  # Start the job periodically after time elapsed
+  # for example: "1m" (execute query every minute)
+  #
+  # There is no period by default. If no period is given, then the statement is run
+  # exactly once.
+  config :period, :validate => :string
+
   # Path to file with last run time.
   # The default will write file to `<path.data>/plugins/inputs/jdbc/logstash_jdbc_last_run`
   # NOTE: it must be a file path and not a directory path
@@ -241,6 +255,11 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   def register
     @logger = self.logger
 
+    if [@interval, @schedule, @period].compact.size > 1
+      raise LogStash::ConfigurationError.new("Use only one of: interval, period, schedule.")
+    end
+
+
     if @record_last_run
       if @last_run_metadata_path.nil?
         logstash_data_path = LogStash::SETTINGS.get_value("path.data")
@@ -322,6 +341,12 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
       # scheduler input thread name example: "[my-oracle]|input|jdbc|scheduler"
       scheduler.cron(@schedule) { execute_query(queue) }
       scheduler.join
+    elsif @interval
+      scheduler.interval(@interval) { execute_query(queue) }
+      scheduler.join
+    elsif @period
+      scheduler.every(@period) { execute_query(queue) }
+      scheduler.join
     else
       execute_query(queue)
     end

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.5.3'
+  s.version         = '5.6.1'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/jdbc/read_write_database_spec.rb

@@ -19,8 +19,13 @@ module LogStash module Filters module Jdbc
           stub_driver_class = double('org.apache.derby.jdbc.EmbeddedDriver').as_null_object
           expect(::Sequel::JDBC).to receive(:load_driver).once.with("org.apache.derby.jdbc.EmbeddedDriver").and_return(stub_driver_class)
           # two calls to connect because ReadWriteDatabase does verify_connection and connect
-          expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {:driver => stub_driver_class, :test => true}).and_return(db)
-          expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {:driver => stub_driver_class}).and_return(db)
+          connection_options = {
+            driver: stub_driver_class,
+            max_connections: 16,
+            pool_timeout: 30,
+          }
+          expect(::Sequel).to receive(:connect).once.with(connection_string_regex, connection_options.merge(:test => true)).and_return(db)
+          expect(::Sequel).to receive(:connect).once.with(connection_string_regex, connection_options).and_return(db)
           expect(read_write_db.empty_record_set).to eq([])
         end
 
@@ -30,8 +35,17 @@ module LogStash module Filters module Jdbc
           password = Util::Password.new("secret")
           stub_driver_class = double('com.example.Driver')
           expect(::Sequel::JDBC).to receive(:load_driver).once.with("a driver class").and_return(stub_driver_class)
-          expect(::Sequel).to receive(:connect).once.with(connection_str, {:driver => stub_driver_class, :user => user, :password => password.value, :test => true}).and_return(db)
-          expect(::Sequel).to receive(:connect).once.with(connection_str, {:driver => stub_driver_class, :user => user, :password => password.value}).and_return(db)
+
+          connection_options = {
+            driver: stub_driver_class,
+            user: user,
+            password: password.value,
+            max_connections: 16,
+            pool_timeout: 30,
+          }
+
+          expect(::Sequel).to receive(:connect).once.with(connection_str, connection_options.merge(:test => true)).and_return(db)
+          expect(::Sequel).to receive(:connect).once.with(connection_str, connection_options).and_return(db)
           described_class.create(connection_str, "a driver class", nil, user, password)
         end
       end

data/spec/inputs/jdbc_spec.rb

@@ -228,6 +228,33 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  describe "scheduling options" do
+    let(:settings) { super().merge("statement" => "SELECT :num_param as num_param FROM SYSIBM.SYSDUMMY1") }
+    scheduling_options = ["interval", "schedule", "period"]
+    scheduling_options.combination(2).each do |option1, option2|
+      context "when using '#{option1}' and '#{option2}' at the same time" do
+        let(:settings) { super().merge(option1 => 'a', option2 => 'b') }
+        it "raises a configuration error" do
+          expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Use only one/)
+        end
+      end
+    end
+    context "when using 'schedule', 'period' and 'interval' at the same time" do
+      let(:settings) { super().merge("interval" => "a", "period" => "b", "schedule" => "c") }
+      it "raises a configuration error" do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Use only one/)
+      end
+    end
+    scheduling_options.each do |option|
+      context "when using only '#{option}'" do
+        let(:settings) { super().merge(option => "a") }
+        it "does not raise a configuration error" do
+          expect { plugin.register }.to_not raise_error
+        end
+      end
+    end
+  end
+
   context "when scheduling" do
     let(:settings) { {"statement" => "SELECT 1 as num_param FROM SYSIBM.SYSDUMMY1", "schedule" => "* * * * * UTC"} }
 

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.5.3
+  version: 5.6.1
 platform: ruby
 authors:
 - Elastic
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-07 00:00:00.000000000 Z
+date: 2025-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,7 +19,6 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -31,12 +30,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 6.5.0
-  name: logstash-core
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -45,12 +44,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 6.5.0
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -59,12 +58,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.74.0
-  name: sequel
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -73,12 +72,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.74.0
 - !ruby/object:Gem::Dependency
+  name: lru_redux
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: lru_redux
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -87,12 +86,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: tzinfo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: tzinfo
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -101,12 +100,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: tzinfo-data
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: tzinfo-data
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -115,12 +114,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ecs_compatibility_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-  name: logstash-mixin-ecs_compatibility_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -129,12 +128,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-validator_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-validator_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -143,12 +142,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-event_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-event_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -157,12 +156,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-scheduler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-scheduler
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -171,12 +170,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: childprocess
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: childprocess
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -185,12 +184,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.3'
-  name: logstash-devutils
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -199,12 +198,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.3'
 - !ruby/object:Gem::Dependency
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: timecop
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -294,7 +293,6 @@ metadata:
   logstash_plugin: 'true'
   logstash_group: integration
   integration_plugins: logstash-input-jdbc,logstash-filter-jdbc_streaming,logstash-filter-jdbc_static
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -310,8 +308,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Integration with JDBC - input and filter plugins
 test_files: