logstash-integration-jdbc 5.5.1 → 5.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4ba7e1b0f0a043e0b58ff727a16decd9f5fb63ac5cc68a1c935af2d59c0235a3
-  data.tar.gz: fa9178574ca2acce0b1f8f0bab0823649e202af2d3911985fb9368a8c3f60c11
+  metadata.gz: a28e2a3a37342bc631ed300da7fa687e99fc879d8cc599669d64e2cfc6418177
+  data.tar.gz: 38b675462f29af81148dbdcc58144a2e3df932b6c6a44825b580a3468ac90acb
 SHA512:
-  metadata.gz: 681d44e0f888cd9c7e87cd33551aa62c5aba9524e714af9db334286b41947c0683dcda15b3fb0e8a8b903976599e261dd0c805666db977ed16f6a145f966c507
-  data.tar.gz: d7fb56739d0e2db3751b2956f0e47dfc585173ff7703cb5b9c0d52b438b10358a6678c72267fa3103cd92ef6137da5376c7398bbe89289a665b7a224a2e9a2d8
+  metadata.gz: b3af9d1d0c7ddc88b8102d7fafb96a9c379a734266698a3136e9e9522f640f64f4f019ea5820c9bc382a460df5f47f2fc62bb6cf8c3ad47ad84f5069710831cd
+  data.tar.gz: 41a6e399b350068d1ed337c158b94f9457f0ace2f31588888db8c99b9e20a39f7fe0f06ab4e809cd01c7a635c9ede2a00629aa1ecf04b9cd86951fdda5c9a828

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.5.3
+  - [DOC] Rework inline comment to a callout in preparation for upcoming MD conversion [#181](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/181)
+
+## 5.5.2
+  - FIX: the input plugin's prior behaviour of opening a new database connection for each scheduled run (removed in `v5.4.1`) is restored, ensuring that infrequently-run schedules do not hold open connections to their databases indefinitely, _without_ reintroducing the leak [#130](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/130)
+
 ## 5.5.1
   - Document `statement_retry_attempts` and `statement_retry_attempts_wait_time` options
 

data/docs/filter-jdbc_static.asciidoc

@@ -123,7 +123,7 @@ filter {
     add_field => { user_lastname => "%{[user][0][lastname]}" }
     remove_field => ["server", "user"]
     staging_directory => "/tmp/logstash/jdbc_static/import_data"
-    loader_schedule => "* */2 * * *" # run loaders every 2 hours
+    loader_schedule => "* */2 * * *" <8>
     jdbc_user => "logstash"
     jdbc_password => "example"
     jdbc_driver_class => "org.postgresql.Driver"
@@ -153,6 +153,7 @@ returns multiple columns, the data is stored as a JSON object within the field.
 <6> When the user is not found in the database, an event is created using data from the <<plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<plugins-{type}s-{plugin}-tag_on_default_use>>.
 <7> Takes data from the JSON object and stores it in top-level event fields for
 easier analysis in Kibana.
+<8> Runs loaders every 2 hours.
 
 Here's a full example:
 

data/lib/logstash/inputs/jdbc.rb

@@ -261,6 +261,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
       end
     end
 
+    prepare_jdbc_connection
+
     if @use_column_value
       # Raise an error if @use_column_value is true, but no @tracking_column is set
       if @tracking_column.nil?
@@ -303,20 +305,7 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
         converters[encoding] = converter
       end
     end
-
     load_driver
-    begin
-      open_jdbc_connection
-    rescue Sequel::DatabaseConnectionError,
-           Sequel::DatabaseError,
-           Sequel::InvalidValue,
-           Java::JavaSql::SQLException => e
-      details = { exception: e.class, message: e.message }
-      details[:cause] = e.cause.inspect if e.cause
-      details[:backtrace] = e.backtrace if @logger.debug?
-      @logger.warn("Exception when executing JDBC query", details)
-      raise(LogStash::ConfigurationError, "Can't create a connection pool to the database")
-    end
   end # def register
 
   # test injection points

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -3,15 +3,17 @@
 require "logstash/config/mixin"
 require "time"
 require "date"
+require "thread" # Monitor
 require_relative "value_tracking"
 require_relative "timezone_proxy"
 require_relative "statement_handler"
 require_relative "value_handler"
 
-java_import java.util.concurrent.locks.ReentrantLock
-
 # Tentative of abstracting JDBC logic to a mixin
 # for potential reuse in other plugins (input/output)
+#
+# CAUTION: implementation of this "potential reuse" module is
+#          VERY tightly-coupled with the JDBC Input's implementation.
 module LogStash  module PluginMixins module Jdbc
   module Jdbc
     include LogStash::PluginMixins::Jdbc::ValueHandler
@@ -159,82 +161,94 @@ module LogStash  module PluginMixins module Jdbc
     end
 
     def open_jdbc_connection
-      # at this point driver is already loaded
-      Sequel.application_timezone = @plugin_timezone.to_sym
-
-      @database = jdbc_connect()
-      @database.extension(:pagination)
-      if @jdbc_default_timezone
-        @database.extension(:named_timezones)
-        @database.timezone = TimezoneProxy.load(@jdbc_default_timezone)
-      end
-      if @jdbc_validate_connection
-        @database.extension(:connection_validator)
-        @database.pool.connection_validation_timeout = @jdbc_validation_timeout
-      end
-      @database.fetch_size = @jdbc_fetch_size unless @jdbc_fetch_size.nil?
-      begin
-        @database.test_connection
-      rescue Java::JavaSql::SQLException => e
-        @logger.warn("Failed test_connection with java.sql.SQLException.", :exception => e)
-      rescue Sequel::DatabaseConnectionError => e
-        @logger.warn("Failed test_connection.", :exception => e)
-        #TODO return false and let the plugin raise a LogStash::ConfigurationError
-        raise e
-      end
+      @connection_lock.synchronize do
+        # at this point driver is already loaded
+        Sequel.application_timezone = @plugin_timezone.to_sym
+
+        @database = jdbc_connect()
+        @database.extension(:pagination)
+        if @jdbc_default_timezone
+          @database.extension(:named_timezones)
+          @database.timezone = TimezoneProxy.load(@jdbc_default_timezone)
+        end
+        if @jdbc_validate_connection
+          @database.extension(:connection_validator)
+          @database.pool.connection_validation_timeout = @jdbc_validation_timeout
+        end
+        @database.fetch_size = @jdbc_fetch_size unless @jdbc_fetch_size.nil?
+        begin
+          @database.test_connection
+        rescue Java::JavaSql::SQLException => e
+          @logger.warn("Failed test_connection with java.sql.SQLException.", :exception => e)
+        rescue Sequel::DatabaseConnectionError => e
+          @logger.warn("Failed test_connection.", :exception => e)
+          #TODO return false and let the plugin raise a LogStash::ConfigurationError
+          raise e
+        end
 
-      @database.sql_log_level = @sql_log_level.to_sym
-      @database.logger = @logger
+        @database.sql_log_level = @sql_log_level.to_sym
+        @database.logger = @logger
 
-      @database.extension :identifier_mangling
+        @database.extension :identifier_mangling
 
-      if @lowercase_column_names
-        @database.identifier_output_method = :downcase
-      else
-        @database.identifier_output_method = :to_s
+        if @lowercase_column_names
+          @database.identifier_output_method = :downcase
+        else
+          @database.identifier_output_method = :to_s
+        end
       end
     end
 
+    public
+    def prepare_jdbc_connection
+      @connection_lock = Monitor.new # aka ReentrantLock
+    end
+
     public
     def close_jdbc_connection
-      begin
+      @connection_lock.synchronize do
         # pipeline restarts can also close the jdbc connection, block until the current executing statement is finished to avoid leaking connections
         # connections in use won't really get closed
         @database.disconnect if @database
-      rescue => e
-        @logger.warn("Failed to close connection", :exception => e)
       end
+    rescue => e
+      @logger.warn("Failed to close connection", :exception => e)
     end
 
     public
-    def execute_statement
+    def execute_statement(&result_handler)
       success = false
       retry_attempts = @statement_retry_attempts
 
-      begin
-        retry_attempts -= 1
-        sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
-        @tracking_column_warning_sent = false
-        @statement_handler.perform_query(@database, @value_tracker.value) do |row|
-          sql_last_value = get_column_value(row) if @use_column_value
-          yield extract_values_from(row)
-        end
-        success = true
-      rescue Sequel::Error, Java::JavaSql::SQLException => e
-        details = { exception: e.class, message: e.message }
-        details[:cause] = e.cause.inspect if e.cause
-        details[:backtrace] = e.backtrace if @logger.debug?
-        @logger.warn("Exception when executing JDBC query", details)
-
-        if retry_attempts == 0
-          @logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
+      @connection_lock.synchronize do
+        begin
+          retry_attempts -= 1
+          open_jdbc_connection
+          sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
+          @tracking_column_warning_sent = false
+          @statement_handler.perform_query(@database, @value_tracker.value) do |row|
+            sql_last_value = get_column_value(row) if @use_column_value
+            yield extract_values_from(row)
+          end
+          success = true
+        rescue Sequel::Error, Java::JavaSql::SQLException => e
+          details = { exception: e.class, message: e.message }
+          details[:cause] = e.cause.inspect if e.cause
+          details[:backtrace] = e.backtrace if @logger.debug?
+          @logger.warn("Exception when executing JDBC query", details)
+
+          if retry_attempts == 0
+            @logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
+          else
+            @logger.error("Unable to execute statement. Trying again.")
+            sleep(@statement_retry_attempts_wait_time)
+            retry
+          end
         else
-          @logger.error("Unable to execute statement. Trying again.")
-          sleep(@statement_retry_attempts_wait_time)
-          retry
+          @value_tracker.set_value(sql_last_value)
+        ensure
+          close_jdbc_connection
         end
-      else
-        @value_tracker.set_value(sql_last_value)
       end
 
       return success

data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb

@@ -124,14 +124,14 @@ module LogStash module PluginMixins module Jdbc
   end
 
   class PreparedStatementHandler < StatementHandler
-    attr_reader :name, :bind_values_array, :statement_prepared, :prepared, :parameters
+    attr_reader :name
 
     def initialize(plugin)
       super(plugin)
       @name = plugin.prepared_statement_name.to_sym
-      @bind_values_array = plugin.prepared_statement_bind_values
-      @parameters = plugin.parameters
-      @statement_prepared = Concurrent::AtomicBoolean.new(false)
+
+      @positional_bind_mapping =  create_positional_bind_mapping(plugin.prepared_statement_bind_values).freeze
+      @positional_bind_placeholders = @positional_bind_mapping.keys.map { |v| :"$#{v}" }.freeze
     end
 
     # Performs the query, ignoring our pagination settings, yielding once per row of data
@@ -148,41 +148,28 @@ module LogStash module PluginMixins module Jdbc
     private
 
     def build_query(db, sql_last_value)
-      @parameters = create_bind_values_hash
-      if statement_prepared.false?
-        prepended = parameters.keys.map{|v| v.to_s.prepend("$").to_sym}
-        @prepared = db[statement, *prepended].prepare(:select, name)
-        statement_prepared.make_true
-      end
       # under the scheduler the Sequel database instance is recreated each time
       # so the previous prepared statements are lost, add back
-      if db.prepared_statement(name).nil?
-        db.set_prepared_statement(name, prepared)
-      end
-      bind_value_sql_last_value(sql_last_value)
-      begin
-        db.call(name, parameters)
-      rescue => e
-        # clear the statement prepared flag - the statement may be closed by this
-        # time.
-        statement_prepared.make_false
-        raise e
-      end
+      prepared = db.prepared_statement(name)
+      prepared ||= db[statement, *positional_bind_placeholders].prepare(:select, name)
+
+      prepared.call(positional_bind_mapping(sql_last_value))
     end
 
-    def create_bind_values_hash
+    def create_positional_bind_mapping(bind_values_array)
       hash = {}
       bind_values_array.each_with_index {|v,i| hash[:"p#{i}"] = v}
       hash
     end
 
-    def bind_value_sql_last_value(sql_last_value)
-      parameters.keys.each do |key|
-        value = parameters[key]
-        if value == ":sql_last_value"
-          parameters[key] = sql_last_value
-        end
+    def positional_bind_mapping(sql_last_value)
+      @positional_bind_mapping.transform_values do |value|
+        value == ":sql_last_value" ? sql_last_value : value
       end
     end
+
+    def positional_bind_placeholders
+      @positional_bind_mapping.keys.map { |v| :"$#{v}" }.freeze
+    end
   end
 end end end

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.5.1'
+  s.version         = '5.5.3'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/integration/integ_spec.rb

@@ -113,13 +113,16 @@ describe LogStash::Inputs::Jdbc, :integration => true do
     end
 
     it "log warning msg when plugin run" do
+      plugin.register
       expect( plugin ).to receive(:log_java_exception)
       expect(plugin.logger).to receive(:warn).once.with("Exception when executing JDBC query",
                                                         hash_including(:message => instance_of(String)))
-      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
+      q = Queue.new
+      expect{ plugin.run(q) }.not_to raise_error
     end
 
     it "should log (native) Java driver error" do
+      plugin.register
       expect( org.apache.logging.log4j.LogManager ).to receive(:getLogger).and_wrap_original do |m, *args|
         logger = m.call(*args)
         expect( logger ).to receive(:error) do |_, e|
@@ -127,7 +130,8 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         end.and_call_original
         logger
       end
-      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
+      q = Queue.new
+      expect{ plugin.run(q) }.not_to raise_error
     end
   end
 end

data/spec/inputs/jdbc_spec.rb

@@ -1283,6 +1283,7 @@ describe LogStash::Inputs::Jdbc do
       plugin.register
       plugin.run(queue)
       db = plugin.instance_variable_get(:@database)
+      expect(db.pool).to be_a_kind_of(::Sequel::ThreadedConnectionPool) # pries into internal details
       expect(db.pool.instance_variable_get(:@timeout)).to eq(0)
       expect(db.pool.instance_variable_get(:@max_size)).to eq(1)
 
@@ -1296,11 +1297,12 @@ describe LogStash::Inputs::Jdbc do
 
     it "should log error message" do
       allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
-      expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 1 times.")
-      expect do
-        plugin.register
-        plugin.run(queue)
-      end.to raise_error(Sequel::PoolTimeout)
+      allow(plugin.logger).to receive(:error)
+
+      plugin.register
+      plugin.run(queue)
+
+      expect(plugin.logger).to have_received(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 1 times.")
     end
   end
 
@@ -1376,12 +1378,13 @@ describe LogStash::Inputs::Jdbc do
       mixin_settings['connection_retry_attempts'] = 2
       mixin_settings['jdbc_pool_timeout'] = 0
       allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
-      expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Trying again.")
-      expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 2 times.")
-      expect do
-        plugin.register
-        plugin.run(queue)
-      end.to raise_error(Sequel::PoolTimeout)
+      allow(plugin.logger).to receive(:error)
+
+      plugin.register
+      plugin.run(queue)
+
+      expect(plugin.logger).to have_received(:error).with("Failed to connect to database. 0 second timeout exceeded. Trying again.")
+      expect(plugin.logger).to have_received(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 2 times.")
     end
 
     it "should not fail when passed a non-positive value" do
@@ -1642,16 +1645,12 @@ describe LogStash::Inputs::Jdbc do
       { "statement" => "SELECT * from types_table", "jdbc_driver_library" => invalid_driver_jar_path }
     end
 
-    after do
-      plugin.stop
-    end
-
-    it "raise a loading error" do
+    it "raise a loading error during #register" do
       expect(File.exists?(invalid_driver_jar_path)).to be true
       expect(FileTest.readable?(invalid_driver_jar_path)).to be false
 
-      expect { plugin.register }.
-          to raise_error(LogStash::PluginLoadingError, /unable to load .*? from :jdbc_driver_library, file not readable/)
+      expect { plugin.register }
+        .to raise_error(LogStash::PluginLoadingError, /unable to load .*? from :jdbc_driver_library, file not readable/)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.5.1
+  version: 5.5.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-03 00:00:00.000000000 Z
+date: 2025-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement