logstash-integration-jdbc 5.4.1 → 5.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71bdfca5d1e5e86b7d86b5c49caf237cfc882e31dab78d63e169241e1c96814d
-  data.tar.gz: 5c07e44042c4a6e6e29b8ecc7fb7631147100a27777c6fdd2a22015ac7fd3b4e
+  metadata.gz: c54f1c22363c2d0637b44a2b68a98e22a1b2f07bc68927a4cc9bd6788a6fc009
+  data.tar.gz: 3ea76ee4fcdde152e43af6c0aab53be50dc022a8c31fce8ebac36669370a4875
 SHA512:
-  metadata.gz: 53e45e47383b15f559e58969e96c0217c43ceb5ba3528db5d6ce3bad10628da6e4c14442aeb5a8795e17b667ac558e5b1377134bb0ecc5e60a375e6ebe106b2c
-  data.tar.gz: d33a2cdea75e56fbe338e175276679e2910b4023c956bc8df7d0795f58140b39da1b9167d668c7665e27b2577c086409b3e390b2a3f8d86c4f9a3e3cdef343e3
+  metadata.gz: d31cef0d20064897337825bcf7ff491d3b64295c47b644d527974f458ac2c8a180d930b84b55fac6601a5f76faa10754b8975af681c4e0718ec2711e08a0d571
+  data.tar.gz: 6434512f2b958b976a12a7c4afe3d584066cb505c4818d8e340e363a7db4327a433785fc9ddfddc6c67eb2d719fc3f518346974760003873250895cb396d36e1

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 5.4.2
+  - Doc: described default_hash and tag_on_default_use interaction filter plugin [#122](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/122)
+  - Added new settings `statement_retry_attempts` and `statement_retry_attempts_wait_time` for retry of failed sql statement execution [#123](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/123)
+
 ## 5.4.1
   - Bugfix leak which happened in creating a new Database pool for every query. The pool is now crated on registration and closed on plugin's `stop` [#119](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/119)
  

data/docs/filter-jdbc_static.asciidoc

@@ -111,10 +111,14 @@ filter {
         query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?"
         prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4>
         target => "user" <5>
+        default_hash => { <6>
+          firstname => nil
+          lastname => nil
+        }
       }
     ]
     # using add_field here to add & rename values to the event root
-    add_field => { server_name => "%{[server][0][description]}" } <6>
+    add_field => { server_name => "%{[server][0][description]}" } <7>
     add_field => { user_firstname => "%{[user][0][firstname]}" }
     add_field => { user_lastname => "%{[user][0][lastname]}" }
     remove_field => ["server", "user"]
@@ -127,6 +131,13 @@ filter {
     jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2"
   }
 }
+
+output {
+  if "_jdbcstaticdefaultsused" in [tags] {
+    # Print all the not found users
+    stdout { }
+  }
+}
 -----
 <1> Queries an external database to fetch the dataset that will be cached
 locally.
@@ -139,7 +150,8 @@ See <<plugins-{type}s-{plugin}-object_order>>.
 follow the positional ordering.
 <5> Specifies the event field that will store the looked-up data. If the lookup
 returns multiple columns, the data is stored as a JSON object within the field.
-<6> Takes data from the JSON object and stores it in top-level event fields for
+<6> When the user is not found in the database, an event is created using data from the <<plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<plugins-{type}s-{plugin}-tag_on_default_use>>.
+<7> Takes data from the JSON object and stores it in top-level event fields for
 easier analysis in Kibana.
 
 Here's a full example:

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -104,6 +104,11 @@ module LogStash  module PluginMixins module Jdbc
       # Number of seconds to sleep between connection attempts
       config :connection_retry_attempts_wait_time, :validate => :number, :default => 0.5
 
+      # Maximum number of times to try running statement
+      config :statement_retry_attempts, :validate => :number, :default => 1
+      # Number of seconds to sleep between statement execution
+      config :statement_retry_attempts_wait_time, :validate => :number, :default => 0.5
+
       # give users the ability to force Sequel application side into using local timezone
       config :plugin_timezone, :validate => ["local", "utc"], :default => "utc"
     end
@@ -202,7 +207,10 @@ module LogStash  module PluginMixins module Jdbc
     public
     def execute_statement
       success = false
+      retry_attempts = @statement_retry_attempts
+
       begin
+        retry_attempts -= 1
         sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
         @tracking_column_warning_sent = false
         @statement_handler.perform_query(@database, @value_tracker.value) do |row|
@@ -210,17 +218,23 @@ module LogStash  module PluginMixins module Jdbc
           yield extract_values_from(row)
         end
         success = true
-      rescue Sequel::DatabaseConnectionError,
-             Sequel::DatabaseError,
-             Sequel::InvalidValue,
-             Java::JavaSql::SQLException => e
+      rescue Sequel::Error, Java::JavaSql::SQLException => e
         details = { exception: e.class, message: e.message }
         details[:cause] = e.cause.inspect if e.cause
         details[:backtrace] = e.backtrace if @logger.debug?
         @logger.warn("Exception when executing JDBC query", details)
+
+        if retry_attempts == 0
+          @logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
+        else
+          @logger.error("Unable to execute statement. Trying again.")
+          sleep(@statement_retry_attempts_wait_time)
+          retry
+        end
       else
         @value_tracker.set_value(sql_last_value)
       end
+
       return success
     end
 

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.4.1'
+  s.version         = '5.4.2'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -1352,6 +1352,34 @@ describe LogStash::Inputs::Jdbc do
       expect { plugin.register }.to_not raise_error
       plugin.stop
     end
+
+    it "does retry when query execution fails" do
+      mixin_settings['statement_retry_attempts'] = 2
+      mixin_settings['statement_retry_attempts_wait_time'] = 0.5
+      queue = Queue.new
+      plugin.register
+
+      handler = plugin.instance_variable_get(:@statement_handler)
+      allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_raise(Sequel::PoolTimeout)
+      expect(plugin.logger).to receive(:error).with("Unable to execute statement. Trying again.")
+      expect(plugin.logger).to receive(:error).with("Unable to execute statement. Tried 2 times.")
+
+      plugin.run(queue)
+      plugin.stop
+    end
+
+    it "does not retry when query execution succeeds" do
+      mixin_settings['connection_retry_attempts'] = 2
+      queue = Queue.new
+      plugin.register
+
+      handler = plugin.instance_variable_get(:@statement_handler)
+      allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_call_original
+      expect(plugin.logger).not_to receive(:error)
+
+      plugin.run(queue)
+      plugin.stop
+    end
   end
 
   context "when encoding of some columns need to be changed" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.4.1
+  version: 5.4.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-04 00:00:00.000000000 Z
+date: 2023-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement