logstash-integration-jdbc 5.4.0 → 5.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28cba197157488c1839fc9a948c3f27916b26af07f28c592f537434bd84722a6
-  data.tar.gz: fe2263ea7ed36abdeb9fd3d9597958724be77a8653950e9d554a212c1e3dd106
+  metadata.gz: c54f1c22363c2d0637b44a2b68a98e22a1b2f07bc68927a4cc9bd6788a6fc009
+  data.tar.gz: 3ea76ee4fcdde152e43af6c0aab53be50dc022a8c31fce8ebac36669370a4875
 SHA512:
-  metadata.gz: 76543ead6834631efaca25d154abe7ee7594943dff0c27d90e7588ed3aa651de427fc3372cffac348f6f810f11cfe13864d8f26af7d09fd34470491a6a4c66b3
-  data.tar.gz: de71ae5f8c54dfa08d67e1e848e59fc1779477c41246cee120debf633301829e931b7103546de0bef482002eb7b5296b748d6ad6caf0bb5457abe7205db79b6e
+  metadata.gz: d31cef0d20064897337825bcf7ff491d3b64295c47b644d527974f458ac2c8a180d930b84b55fac6601a5f76faa10754b8975af681c4e0718ec2711e08a0d571
+  data.tar.gz: 6434512f2b958b976a12a7c4afe3d584066cb505c4818d8e340e363a7db4327a433785fc9ddfddc6c67eb2d719fc3f518346974760003873250895cb396d36e1

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 5.4.2
+  - Doc: described default_hash and tag_on_default_use interaction filter plugin [#122](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/122)
+  - Added new settings `statement_retry_attempts` and `statement_retry_attempts_wait_time` for retry of failed sql statement execution [#123](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/123)
+
+## 5.4.1
+  - Bugfix leak which happened in creating a new Database pool for every query. The pool is now crated on registration and closed on plugin's `stop` [#119](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/119)
+ 
 ## 5.4.0
   - Ambiguous Timestamp Support [#92](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/92)
     - FIX: when encountering an ambiguous timestamp, the JDBC Input no longer crashes

data/docs/filter-jdbc_static.asciidoc

@@ -111,10 +111,14 @@ filter {
         query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?"
         prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4>
         target => "user" <5>
+        default_hash => { <6>
+          firstname => nil
+          lastname => nil
+        }
       }
     ]
     # using add_field here to add & rename values to the event root
-    add_field => { server_name => "%{[server][0][description]}" } <6>
+    add_field => { server_name => "%{[server][0][description]}" } <7>
     add_field => { user_firstname => "%{[user][0][firstname]}" }
     add_field => { user_lastname => "%{[user][0][lastname]}" }
     remove_field => ["server", "user"]
@@ -127,6 +131,13 @@ filter {
     jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2"
   }
 }
+
+output {
+  if "_jdbcstaticdefaultsused" in [tags] {
+    # Print all the not found users
+    stdout { }
+  }
+}
 -----
 <1> Queries an external database to fetch the dataset that will be cached
 locally.
@@ -139,7 +150,8 @@ See <<plugins-{type}s-{plugin}-object_order>>.
 follow the positional ordering.
 <5> Specifies the event field that will store the looked-up data. If the lookup
 returns multiple columns, the data is stored as a JSON object within the field.
-<6> Takes data from the JSON object and stores it in top-level event fields for
+<6> When the user is not found in the database, an event is created using data from the <<plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<plugins-{type}s-{plugin}-tag_on_default_use>>.
+<7> Takes data from the JSON object and stores it in top-level event fields for
 easier analysis in Kibana.
 
 Here's a full example:

data/lib/logstash/inputs/jdbc.rb

@@ -261,8 +261,6 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
       end
     end
 
-    prepare_jdbc_connection
-
     if @use_column_value
       # Raise an error if @use_column_value is true, but no @tracking_column is set
       if @tracking_column.nil?
@@ -305,6 +303,20 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
         converters[encoding] = converter
       end
     end
+
+    load_driver
+    begin
+      open_jdbc_connection
+    rescue Sequel::DatabaseConnectionError,
+           Sequel::DatabaseError,
+           Sequel::InvalidValue,
+           Java::JavaSql::SQLException => e
+      details = { exception: e.class, message: e.message }
+      details[:cause] = e.cause.inspect if e.cause
+      details[:backtrace] = e.backtrace if @logger.debug?
+      @logger.warn("Exception when executing JDBC query", details)
+      raise(LogStash::ConfigurationError, "Can't create a connection pool to the database")
+    end
   end # def register
 
   # test injection points
@@ -317,7 +329,6 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   end
 
   def run(queue)
-    load_driver
     if @schedule
       # scheduler input thread name example: "[my-oracle]|input|jdbc|scheduler"
       scheduler.cron(@schedule) { execute_query(queue) }

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -104,6 +104,11 @@ module LogStash  module PluginMixins module Jdbc
       # Number of seconds to sleep between connection attempts
       config :connection_retry_attempts_wait_time, :validate => :number, :default => 0.5
 
+      # Maximum number of times to try running statement
+      config :statement_retry_attempts, :validate => :number, :default => 1
+      # Number of seconds to sleep between statement execution
+      config :statement_retry_attempts_wait_time, :validate => :number, :default => 0.5
+
       # give users the ability to force Sequel application side into using local timezone
       config :plugin_timezone, :validate => ["local", "utc"], :default => "utc"
     end
@@ -188,31 +193,24 @@ module LogStash  module PluginMixins module Jdbc
       end
     end
 
-    public
-    def prepare_jdbc_connection
-      @connection_lock = ReentrantLock.new
-    end
-
     public
     def close_jdbc_connection
       begin
         # pipeline restarts can also close the jdbc connection, block until the current executing statement is finished to avoid leaking connections
         # connections in use won't really get closed
-        @connection_lock.lock
         @database.disconnect if @database
       rescue => e
         @logger.warn("Failed to close connection", :exception => e)
-      ensure
-        @connection_lock.unlock
       end
     end
 
     public
     def execute_statement
       success = false
+      retry_attempts = @statement_retry_attempts
+
       begin
-        @connection_lock.lock
-        open_jdbc_connection
+        retry_attempts -= 1
         sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
         @tracking_column_warning_sent = false
         @statement_handler.perform_query(@database, @value_tracker.value) do |row|
@@ -220,20 +218,23 @@ module LogStash  module PluginMixins module Jdbc
           yield extract_values_from(row)
         end
         success = true
-      rescue Sequel::DatabaseConnectionError,
-             Sequel::DatabaseError,
-             Sequel::InvalidValue,
-             Java::JavaSql::SQLException => e
+      rescue Sequel::Error, Java::JavaSql::SQLException => e
         details = { exception: e.class, message: e.message }
         details[:cause] = e.cause.inspect if e.cause
         details[:backtrace] = e.backtrace if @logger.debug?
         @logger.warn("Exception when executing JDBC query", details)
+
+        if retry_attempts == 0
+          @logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
+        else
+          @logger.error("Unable to execute statement. Trying again.")
+          sleep(@statement_retry_attempts_wait_time)
+          retry
+        end
       else
         @value_tracker.set_value(sql_last_value)
-      ensure
-        close_jdbc_connection
-        @connection_lock.unlock
       end
+
       return success
     end
 

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.4.0'
+  s.version         = '5.4.2'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/integration/integ_spec.rb

@@ -76,10 +76,8 @@ describe LogStash::Inputs::Jdbc, :integration => true do
     end
 
     it "should not register correctly" do
-      plugin.register
-      q = Queue.new
       expect do
-        plugin.run(q)
+        plugin.register
       end.to raise_error(::LogStash::PluginLoadingError)
     end
   end
@@ -92,16 +90,13 @@ describe LogStash::Inputs::Jdbc, :integration => true do
     end
 
     it "log warning msg when plugin run" do
-      plugin.register
       expect( plugin ).to receive(:log_java_exception)
       expect(plugin.logger).to receive(:warn).once.with("Exception when executing JDBC query",
                                                         hash_including(:message => instance_of(String)))
-      q = Queue.new
-      expect{ plugin.run(q) }.not_to raise_error
+      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
     end
 
     it "should log (native) Java driver error" do
-      plugin.register
       expect( org.apache.logging.log4j.LogManager ).to receive(:getLogger).and_wrap_original do |m, *args|
         logger = m.call(*args)
         expect( logger ).to receive(:error) do |_, e|
@@ -109,8 +104,7 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         end.and_call_original
         logger
       end
-      q = Queue.new
-      expect{ plugin.run(q) }.not_to raise_error
+      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
     end
   end
 end

data/spec/inputs/jdbc_spec.rb

@@ -1352,6 +1352,34 @@ describe LogStash::Inputs::Jdbc do
       expect { plugin.register }.to_not raise_error
       plugin.stop
     end
+
+    it "does retry when query execution fails" do
+      mixin_settings['statement_retry_attempts'] = 2
+      mixin_settings['statement_retry_attempts_wait_time'] = 0.5
+      queue = Queue.new
+      plugin.register
+
+      handler = plugin.instance_variable_get(:@statement_handler)
+      allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_raise(Sequel::PoolTimeout)
+      expect(plugin.logger).to receive(:error).with("Unable to execute statement. Trying again.")
+      expect(plugin.logger).to receive(:error).with("Unable to execute statement. Tried 2 times.")
+
+      plugin.run(queue)
+      plugin.stop
+    end
+
+    it "does not retry when query execution succeeds" do
+      mixin_settings['connection_retry_attempts'] = 2
+      queue = Queue.new
+      plugin.register
+
+      handler = plugin.instance_variable_get(:@statement_handler)
+      allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_call_original
+      expect(plugin.logger).not_to receive(:error)
+
+      plugin.run(queue)
+      plugin.stop
+    end
   end
 
   context "when encoding of some columns need to be changed" do
@@ -1565,16 +1593,12 @@ describe LogStash::Inputs::Jdbc do
       { "statement" => "SELECT * from types_table", "jdbc_driver_library" => invalid_driver_jar_path }
     end
 
-    before do
-      plugin.register
-    end
-
     after do
       plugin.stop
     end
 
     it "raise a loading error" do
-      expect { plugin.run(queue) }.
+      expect { plugin.register }.
           to raise_error(LogStash::PluginLoadingError, /unable to load .*? from :jdbc_driver_library, file not readable/)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.4.0
+  version: 5.4.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-11 00:00:00.000000000 Z
+date: 2023-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement