logstash-integration-jdbc 5.5.0 → 5.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e155ca5542f41d749481a67cf65cf28ad194f019defe6368f765cd1a0ccf3860
-  data.tar.gz: 81cbc828cbbdecb5d3265c07f5777f68c3103252edbc772868c1aa8f1323b566
+  metadata.gz: 837344b940ff8c7af09a52e7947507aa2431a39feb3562241218bdcdb03b0ae8
+  data.tar.gz: dd89db28ba9ab3ee55d57164d5117dde658f18cb534e0bebec2d357fece044a2
 SHA512:
-  metadata.gz: 0d3f9f07bd92ff1c21f417795e20edbd5f78542181a00f4bfd770dd7f9fb1066df8bbec9af242126b13148049cc70f41190b5246104a23c549af6a62338807c1
-  data.tar.gz: 71c92b8dba18479be7d69f64b580d2bd99902d7d330fd1633997e8134c69403f640e1ba555a1b1640d6737de8b53c68d51e4cab75b4da82eeda7fbf549f72c0c
+  metadata.gz: 5b6bba9b8ad12d504755f60409461236a0b42ae52f91a5503fcd8fa18c4c0ea7d9e900a13d917d3ade818c4332a2cd06e2c1fcf56d34cbe1604712a87b364414
+  data.tar.gz: 5ccb895bdcca3602e93ab9c27f2032131ed71ec763fb77b95dee4ea552d8ec8be695573e997d63875b33e907a94035ec3e01101e69b722a315ff0f5613f18d62

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.5.2
+  - FIX: the input plugin's prior behaviour of opening a new database connection for each scheduled run (removed in `v5.4.1`) is restored, ensuring that infrequently-run schedules do not hold open connections to their databases indefinitely, _without_ reintroducing the leak [#130](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/130)
+
+## 5.5.1
+  - Document `statement_retry_attempts` and `statement_retry_attempts_wait_time` options
+
 ## 5.5.0
   - Feat: add support for SQL `DATE` columns to jdbc static and streaming filters [#171](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/171)
 

data/docs/input-jdbc.asciidoc

@@ -238,6 +238,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-sql_log_level>> |<<string,string>>, one of `["fatal", "error", "warn", "info", "debug"]`|No
 | <<plugins-{type}s-{plugin}-statement>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-statement_retry_attempts>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-statement_retry_attempts_wait_time>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
 | <<plugins-{type}s-{plugin}-tracking_column>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-tracking_column_type>> |<<string,string>>, one of `["numeric", "timestamp"]`|No
@@ -623,6 +625,22 @@ with the `parameters` setting.
 
 Path of file containing statement to execute
 
+[id="plugins-{type}s-{plugin}-statement_retry_attempts"]
+===== `statement_retry_attempts`
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+Maximum number of times to try executing a statement.
+
+[id="plugins-{type}s-{plugin}-statement_retry_attempts_wait_time"]
+===== `statement_retry_attempts_wait_time`
+
+  * Value type is <<number,number>>
+  * Default value is `0.5`
+
+Number of seconds to sleep between statement execution attempts.
+
 [id="plugins-{type}s-{plugin}-target"]
 ===== `target`
 

data/lib/logstash/inputs/jdbc.rb

@@ -261,6 +261,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
       end
     end
 
+    prepare_jdbc_connection
+
     if @use_column_value
       # Raise an error if @use_column_value is true, but no @tracking_column is set
       if @tracking_column.nil?
@@ -303,20 +305,7 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
         converters[encoding] = converter
       end
     end
-
     load_driver
-    begin
-      open_jdbc_connection
-    rescue Sequel::DatabaseConnectionError,
-           Sequel::DatabaseError,
-           Sequel::InvalidValue,
-           Java::JavaSql::SQLException => e
-      details = { exception: e.class, message: e.message }
-      details[:cause] = e.cause.inspect if e.cause
-      details[:backtrace] = e.backtrace if @logger.debug?
-      @logger.warn("Exception when executing JDBC query", details)
-      raise(LogStash::ConfigurationError, "Can't create a connection pool to the database")
-    end
   end # def register
 
   # test injection points

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -3,15 +3,17 @@
 require "logstash/config/mixin"
 require "time"
 require "date"
+require "thread" # Monitor
 require_relative "value_tracking"
 require_relative "timezone_proxy"
 require_relative "statement_handler"
 require_relative "value_handler"
 
-java_import java.util.concurrent.locks.ReentrantLock
-
 # Tentative of abstracting JDBC logic to a mixin
 # for potential reuse in other plugins (input/output)
+#
+# CAUTION: implementation of this "potential reuse" module is
+#          VERY tightly-coupled with the JDBC Input's implementation.
 module LogStash  module PluginMixins module Jdbc
   module Jdbc
     include LogStash::PluginMixins::Jdbc::ValueHandler
@@ -159,82 +161,94 @@ module LogStash  module PluginMixins module Jdbc
     end
 
     def open_jdbc_connection
-      # at this point driver is already loaded
-      Sequel.application_timezone = @plugin_timezone.to_sym
-
-      @database = jdbc_connect()
-      @database.extension(:pagination)
-      if @jdbc_default_timezone
-        @database.extension(:named_timezones)
-        @database.timezone = TimezoneProxy.load(@jdbc_default_timezone)
-      end
-      if @jdbc_validate_connection
-        @database.extension(:connection_validator)
-        @database.pool.connection_validation_timeout = @jdbc_validation_timeout
-      end
-      @database.fetch_size = @jdbc_fetch_size unless @jdbc_fetch_size.nil?
-      begin
-        @database.test_connection
-      rescue Java::JavaSql::SQLException => e
-        @logger.warn("Failed test_connection with java.sql.SQLException.", :exception => e)
-      rescue Sequel::DatabaseConnectionError => e
-        @logger.warn("Failed test_connection.", :exception => e)
-        #TODO return false and let the plugin raise a LogStash::ConfigurationError
-        raise e
-      end
+      @connection_lock.synchronize do
+        # at this point driver is already loaded
+        Sequel.application_timezone = @plugin_timezone.to_sym
+
+        @database = jdbc_connect()
+        @database.extension(:pagination)
+        if @jdbc_default_timezone
+          @database.extension(:named_timezones)
+          @database.timezone = TimezoneProxy.load(@jdbc_default_timezone)
+        end
+        if @jdbc_validate_connection
+          @database.extension(:connection_validator)
+          @database.pool.connection_validation_timeout = @jdbc_validation_timeout
+        end
+        @database.fetch_size = @jdbc_fetch_size unless @jdbc_fetch_size.nil?
+        begin
+          @database.test_connection
+        rescue Java::JavaSql::SQLException => e
+          @logger.warn("Failed test_connection with java.sql.SQLException.", :exception => e)
+        rescue Sequel::DatabaseConnectionError => e
+          @logger.warn("Failed test_connection.", :exception => e)
+          #TODO return false and let the plugin raise a LogStash::ConfigurationError
+          raise e
+        end
 
-      @database.sql_log_level = @sql_log_level.to_sym
-      @database.logger = @logger
+        @database.sql_log_level = @sql_log_level.to_sym
+        @database.logger = @logger
 
-      @database.extension :identifier_mangling
+        @database.extension :identifier_mangling
 
-      if @lowercase_column_names
-        @database.identifier_output_method = :downcase
-      else
-        @database.identifier_output_method = :to_s
+        if @lowercase_column_names
+          @database.identifier_output_method = :downcase
+        else
+          @database.identifier_output_method = :to_s
+        end
       end
     end
 
+    public
+    def prepare_jdbc_connection
+      @connection_lock = Monitor.new # aka ReentrantLock
+    end
+
     public
     def close_jdbc_connection
-      begin
+      @connection_lock.synchronize do
         # pipeline restarts can also close the jdbc connection, block until the current executing statement is finished to avoid leaking connections
         # connections in use won't really get closed
         @database.disconnect if @database
-      rescue => e
-        @logger.warn("Failed to close connection", :exception => e)
       end
+    rescue => e
+      @logger.warn("Failed to close connection", :exception => e)
     end
 
     public
-    def execute_statement
+    def execute_statement(&result_handler)
       success = false
       retry_attempts = @statement_retry_attempts
 
-      begin
-        retry_attempts -= 1
-        sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
-        @tracking_column_warning_sent = false
-        @statement_handler.perform_query(@database, @value_tracker.value) do |row|
-          sql_last_value = get_column_value(row) if @use_column_value
-          yield extract_values_from(row)
-        end
-        success = true
-      rescue Sequel::Error, Java::JavaSql::SQLException => e
-        details = { exception: e.class, message: e.message }
-        details[:cause] = e.cause.inspect if e.cause
-        details[:backtrace] = e.backtrace if @logger.debug?
-        @logger.warn("Exception when executing JDBC query", details)
-
-        if retry_attempts == 0
-          @logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
+      @connection_lock.synchronize do
+        begin
+          retry_attempts -= 1
+          open_jdbc_connection
+          sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
+          @tracking_column_warning_sent = false
+          @statement_handler.perform_query(@database, @value_tracker.value) do |row|
+            sql_last_value = get_column_value(row) if @use_column_value
+            yield extract_values_from(row)
+          end
+          success = true
+        rescue Sequel::Error, Java::JavaSql::SQLException => e
+          details = { exception: e.class, message: e.message }
+          details[:cause] = e.cause.inspect if e.cause
+          details[:backtrace] = e.backtrace if @logger.debug?
+          @logger.warn("Exception when executing JDBC query", details)
+
+          if retry_attempts == 0
+            @logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
+          else
+            @logger.error("Unable to execute statement. Trying again.")
+            sleep(@statement_retry_attempts_wait_time)
+            retry
+          end
         else
-          @logger.error("Unable to execute statement. Trying again.")
-          sleep(@statement_retry_attempts_wait_time)
-          retry
+          @value_tracker.set_value(sql_last_value)
+        ensure
+          close_jdbc_connection
         end
-      else
-        @value_tracker.set_value(sql_last_value)
       end
 
       return success

data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb

@@ -124,14 +124,14 @@ module LogStash module PluginMixins module Jdbc
   end
 
   class PreparedStatementHandler < StatementHandler
-    attr_reader :name, :bind_values_array, :statement_prepared, :prepared, :parameters
+    attr_reader :name
 
     def initialize(plugin)
       super(plugin)
       @name = plugin.prepared_statement_name.to_sym
-      @bind_values_array = plugin.prepared_statement_bind_values
-      @parameters = plugin.parameters
-      @statement_prepared = Concurrent::AtomicBoolean.new(false)
+
+      @positional_bind_mapping =  create_positional_bind_mapping(plugin.prepared_statement_bind_values).freeze
+      @positional_bind_placeholders = @positional_bind_mapping.keys.map { |v| :"$#{v}" }.freeze
     end
 
     # Performs the query, ignoring our pagination settings, yielding once per row of data
@@ -148,41 +148,28 @@ module LogStash module PluginMixins module Jdbc
     private
 
     def build_query(db, sql_last_value)
-      @parameters = create_bind_values_hash
-      if statement_prepared.false?
-        prepended = parameters.keys.map{|v| v.to_s.prepend("$").to_sym}
-        @prepared = db[statement, *prepended].prepare(:select, name)
-        statement_prepared.make_true
-      end
       # under the scheduler the Sequel database instance is recreated each time
       # so the previous prepared statements are lost, add back
-      if db.prepared_statement(name).nil?
-        db.set_prepared_statement(name, prepared)
-      end
-      bind_value_sql_last_value(sql_last_value)
-      begin
-        db.call(name, parameters)
-      rescue => e
-        # clear the statement prepared flag - the statement may be closed by this
-        # time.
-        statement_prepared.make_false
-        raise e
-      end
+      prepared = db.prepared_statement(name)
+      prepared ||= db[statement, *positional_bind_placeholders].prepare(:select, name)
+
+      prepared.call(positional_bind_mapping(sql_last_value))
     end
 
-    def create_bind_values_hash
+    def create_positional_bind_mapping(bind_values_array)
       hash = {}
       bind_values_array.each_with_index {|v,i| hash[:"p#{i}"] = v}
       hash
     end
 
-    def bind_value_sql_last_value(sql_last_value)
-      parameters.keys.each do |key|
-        value = parameters[key]
-        if value == ":sql_last_value"
-          parameters[key] = sql_last_value
-        end
+    def positional_bind_mapping(sql_last_value)
+      @positional_bind_mapping.transform_values do |value|
+        value == ":sql_last_value" ? sql_last_value : value
       end
     end
+
+    def positional_bind_placeholders
+      @positional_bind_mapping.keys.map { |v| :"$#{v}" }.freeze
+    end
   end
 end end end

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.5.0'
+  s.version         = '5.5.2'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,7 +23,6 @@ Gem::Specification.new do |s|
   }
 
   # Gem dependencies
-  s.add_development_dependency 'jar-dependencies', '~> 0.3'
 
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   # Restrict use of this plugin to versions of Logstash where support for integration plugins is present.

data/spec/inputs/integration/integ_spec.rb

@@ -113,13 +113,16 @@ describe LogStash::Inputs::Jdbc, :integration => true do
     end
 
     it "log warning msg when plugin run" do
+      plugin.register
       expect( plugin ).to receive(:log_java_exception)
       expect(plugin.logger).to receive(:warn).once.with("Exception when executing JDBC query",
                                                         hash_including(:message => instance_of(String)))
-      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
+      q = Queue.new
+      expect{ plugin.run(q) }.not_to raise_error
     end
 
     it "should log (native) Java driver error" do
+      plugin.register
       expect( org.apache.logging.log4j.LogManager ).to receive(:getLogger).and_wrap_original do |m, *args|
         logger = m.call(*args)
         expect( logger ).to receive(:error) do |_, e|
@@ -127,7 +130,8 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         end.and_call_original
         logger
       end
-      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
+      q = Queue.new
+      expect{ plugin.run(q) }.not_to raise_error
     end
   end
 end

data/spec/inputs/jdbc_spec.rb

@@ -1283,6 +1283,7 @@ describe LogStash::Inputs::Jdbc do
       plugin.register
       plugin.run(queue)
       db = plugin.instance_variable_get(:@database)
+      expect(db.pool).to be_a_kind_of(::Sequel::ThreadedConnectionPool) # pries into internal details
       expect(db.pool.instance_variable_get(:@timeout)).to eq(0)
       expect(db.pool.instance_variable_get(:@max_size)).to eq(1)
 
@@ -1296,11 +1297,12 @@ describe LogStash::Inputs::Jdbc do
 
     it "should log error message" do
       allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
-      expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 1 times.")
-      expect do
-        plugin.register
-        plugin.run(queue)
-      end.to raise_error(Sequel::PoolTimeout)
+      allow(plugin.logger).to receive(:error)
+
+      plugin.register
+      plugin.run(queue)
+
+      expect(plugin.logger).to have_received(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 1 times.")
     end
   end
 
@@ -1376,12 +1378,13 @@ describe LogStash::Inputs::Jdbc do
       mixin_settings['connection_retry_attempts'] = 2
       mixin_settings['jdbc_pool_timeout'] = 0
       allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
-      expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Trying again.")
-      expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 2 times.")
-      expect do
-        plugin.register
-        plugin.run(queue)
-      end.to raise_error(Sequel::PoolTimeout)
+      allow(plugin.logger).to receive(:error)
+
+      plugin.register
+      plugin.run(queue)
+
+      expect(plugin.logger).to have_received(:error).with("Failed to connect to database. 0 second timeout exceeded. Trying again.")
+      expect(plugin.logger).to have_received(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 2 times.")
     end
 
     it "should not fail when passed a non-positive value" do
@@ -1642,16 +1645,12 @@ describe LogStash::Inputs::Jdbc do
       { "statement" => "SELECT * from types_table", "jdbc_driver_library" => invalid_driver_jar_path }
     end
 
-    after do
-      plugin.stop
-    end
-
-    it "raise a loading error" do
+    it "raise a loading error during #register" do
       expect(File.exists?(invalid_driver_jar_path)).to be true
       expect(FileTest.readable?(invalid_driver_jar_path)).to be false
 
-      expect { plugin.register }.
-          to raise_error(LogStash::PluginLoadingError, /unable to load .*? from :jdbc_driver_library, file not readable/)
+      expect { plugin.register }
+        .to raise_error(LogStash::PluginLoadingError, /unable to load .*? from :jdbc_driver_library, file not readable/)
     end
   end
 

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.5.0
+  version: 5.5.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-09 00:00:00.000000000 Z
+date: 2024-12-23 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  name: jar-dependencies
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -34,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -51,8 +37,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 6.5.0
   name: logstash-core
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -65,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -79,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.74.0
   name: sequel
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -93,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: lru_redux
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -107,8 +93,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: tzinfo
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -121,8 +107,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: tzinfo-data
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -135,8 +121,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
   name: logstash-mixin-ecs_compatibility_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -149,8 +135,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-validator_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -163,8 +149,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-event_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -177,8 +163,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-scheduler
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -191,8 +177,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: childprocess
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -205,8 +191,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.3'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -219,8 +205,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: timecop
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -324,7 +310,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Integration with JDBC - input and filter plugins