logstash-integration-jdbc 5.4.1 → 5.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71bdfca5d1e5e86b7d86b5c49caf237cfc882e31dab78d63e169241e1c96814d
-  data.tar.gz: 5c07e44042c4a6e6e29b8ecc7fb7631147100a27777c6fdd2a22015ac7fd3b4e
+  metadata.gz: 0427bfc9149fd1b4751f91bda69dcbc67fe50bbb59ba331142c52b0ac79b92df
+  data.tar.gz: 9b89e07e05f01b5f4226cbbb452a36e0edb116ca116d843d8f578508206086b8
 SHA512:
-  metadata.gz: 53e45e47383b15f559e58969e96c0217c43ceb5ba3528db5d6ce3bad10628da6e4c14442aeb5a8795e17b667ac558e5b1377134bb0ecc5e60a375e6ebe106b2c
-  data.tar.gz: d33a2cdea75e56fbe338e175276679e2910b4023c956bc8df7d0795f58140b39da1b9167d668c7665e27b2577c086409b3e390b2a3f8d86c4f9a3e3cdef343e3
+  metadata.gz: 7bcc88da6e2d4de082a5547f0f273063913f2fe8e87578d7177eb4929604a66ec8be48818ba7b32a88a7bf978aad42f07c14f69f60349d82531f531c0cd77480
+  data.tar.gz: 76bc8bb2c010a6f6087e91be3280c1b83ae165449889cc425cb91b4e6fe0f9092b229f7ed37df5fb6ae7be3acbaa8c69541c610bbfff07c374a32be014ccdcbf

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 5.4.3
+  - Fix crash when metadata file can't be deleted after moving under path.data [#136](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/136)
+
+## 5.4.2
+  - Doc: described default_hash and tag_on_default_use interaction filter plugin [#122](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/122)
+  - Added new settings `statement_retry_attempts` and `statement_retry_attempts_wait_time` for retry of failed sql statement execution [#123](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/123)
+
 ## 5.4.1
   - Bugfix leak which happened in creating a new Database pool for every query. The pool is now crated on registration and closed on plugin's `stop` [#119](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/119)
  

data/docs/filter-jdbc_static.asciidoc

@@ -111,10 +111,14 @@ filter {
         query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?"
         prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4>
         target => "user" <5>
+        default_hash => { <6>
+          firstname => nil
+          lastname => nil
+        }
       }
     ]
     # using add_field here to add & rename values to the event root
-    add_field => { server_name => "%{[server][0][description]}" } <6>
+    add_field => { server_name => "%{[server][0][description]}" } <7>
     add_field => { user_firstname => "%{[user][0][firstname]}" }
     add_field => { user_lastname => "%{[user][0][lastname]}" }
     remove_field => ["server", "user"]
@@ -127,6 +131,13 @@ filter {
     jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2"
   }
 }
+
+output {
+  if "_jdbcstaticdefaultsused" in [tags] {
+    # Print all the not found users
+    stdout { }
+  }
+}
 -----
 <1> Queries an external database to fetch the dataset that will be cached
 locally.
@@ -139,7 +150,8 @@ See <<plugins-{type}s-{plugin}-object_order>>.
 follow the positional ordering.
 <5> Specifies the event field that will store the looked-up data. If the lookup
 returns multiple columns, the data is stored as a JSON object within the field.
-<6> Takes data from the JSON object and stores it in top-level event fields for
+<6> When the user is not found in the database, an event is created using data from the <<plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<plugins-{type}s-{plugin}-tag_on_default_use>>.
+<7> Takes data from the JSON object and stores it in top-level event fields for
 easier analysis in Kibana.
 
 Here's a full example:

data/docs/input-jdbc.asciidoc

@@ -491,10 +491,12 @@ How often to validate a connection (in seconds)
 ===== `last_run_metadata_path`
 
   * Value type is <<string,string>>
-  * Default value is `"$HOME/.logstash_jdbc_last_run"`
+  * Default value is `"<path.data>/plugins/inputs/jdbc/logstash_jdbc_last_run"`
 
 Path to file with last run time
 
+In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed.
+
 [id="plugins-{type}s-{plugin}-lowercase_column_names"]
 ===== `lowercase_column_names`
 

data/lib/logstash/inputs/jdbc.rb

@@ -406,7 +406,7 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
         # the correct access rights
         ::File.delete(old_default_path.to_path)
         @logger.info("Successfully moved the #{old_default_path.to_path} into #{metadata_new_path.to_path}")
-      rescue e
+      rescue => e
         @logger.warn("Using new metadata file at #{metadata_new_path.to_path} but #{old_default_path} can't be removed.")
       end
     end

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -104,6 +104,11 @@ module LogStash  module PluginMixins module Jdbc
       # Number of seconds to sleep between connection attempts
       config :connection_retry_attempts_wait_time, :validate => :number, :default => 0.5
 
+      # Maximum number of times to try running statement
+      config :statement_retry_attempts, :validate => :number, :default => 1
+      # Number of seconds to sleep between statement execution
+      config :statement_retry_attempts_wait_time, :validate => :number, :default => 0.5
+
       # give users the ability to force Sequel application side into using local timezone
       config :plugin_timezone, :validate => ["local", "utc"], :default => "utc"
     end
@@ -202,7 +207,10 @@ module LogStash  module PluginMixins module Jdbc
     public
     def execute_statement
       success = false
+      retry_attempts = @statement_retry_attempts
+
       begin
+        retry_attempts -= 1
         sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
         @tracking_column_warning_sent = false
         @statement_handler.perform_query(@database, @value_tracker.value) do |row|
@@ -210,17 +218,23 @@ module LogStash  module PluginMixins module Jdbc
           yield extract_values_from(row)
         end
         success = true
-      rescue Sequel::DatabaseConnectionError,
-             Sequel::DatabaseError,
-             Sequel::InvalidValue,
-             Java::JavaSql::SQLException => e
+      rescue Sequel::Error, Java::JavaSql::SQLException => e
         details = { exception: e.class, message: e.message }
         details[:cause] = e.cause.inspect if e.cause
         details[:backtrace] = e.backtrace if @logger.debug?
         @logger.warn("Exception when executing JDBC query", details)
+
+        if retry_attempts == 0
+          @logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
+        else
+          @logger.error("Unable to execute statement. Trying again.")
+          sleep(@statement_retry_attempts_wait_time)
+          retry
+        end
       else
         @value_tracker.set_value(sql_last_value)
       end
+
       return success
     end
 

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.4.1'
+  s.version         = '5.4.3'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -1169,19 +1169,31 @@ describe LogStash::Inputs::Jdbc do
     context "when a file exists" do
       before do
         # in a faked HOME folder save a valid previous last_run metadata file
+        allow(ENV).to receive(:[]).and_call_original
         allow(ENV).to receive(:[]).with('HOME').and_return(fake_home)
-        File.open("#{ENV['HOME']}/.logstash_jdbc_last_run", 'w') do |file|
+
+        File.open("#{fake_home}/.logstash_jdbc_last_run", 'w') do |file|
           file.write("--- !ruby/object:DateTime '2022-03-08 08:10:00.486889000 Z'")
         end
       end
+      let(:old_path) { "#{fake_home}/.logstash_jdbc_last_run" }
+      let(:path_data) { LogStash::SETTINGS.get_value("path.data") }
+      let(:new_path) { "#{path_data}/plugins/inputs/jdbc/logstash_jdbc_last_run" }
 
       it "should be moved" do
         plugin.register
-
-        expect(::File.exist?("#{ENV['HOME']}/.logstash_jdbc_last_run")).to be false
-        path = LogStash::SETTINGS.get_value("path.data")
-        full_path = "#{path}/plugins/inputs/jdbc/logstash_jdbc_last_run"
-        expect(::File.exist?(full_path)).to be true
+        expect(::File).to_not exist(old_path)
+        expect(::File).to exist(new_path)
+      end
+      context "if the delete fails" do
+        before(:each) do
+          allow(File).to receive(:delete).and_raise ArgumentError
+        end
+        it "should be still be moved" do
+          plugin.register
+          expect(::File).to exist(old_path) # old still exists
+          expect(::File).to exist(new_path)
+        end
       end
     end
   end
@@ -1352,6 +1364,34 @@ describe LogStash::Inputs::Jdbc do
       expect { plugin.register }.to_not raise_error
       plugin.stop
     end
+
+    it "does retry when query execution fails" do
+      mixin_settings['statement_retry_attempts'] = 2
+      mixin_settings['statement_retry_attempts_wait_time'] = 0.5
+      queue = Queue.new
+      plugin.register
+
+      handler = plugin.instance_variable_get(:@statement_handler)
+      allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_raise(Sequel::PoolTimeout)
+      expect(plugin.logger).to receive(:error).with("Unable to execute statement. Trying again.")
+      expect(plugin.logger).to receive(:error).with("Unable to execute statement. Tried 2 times.")
+
+      plugin.run(queue)
+      plugin.stop
+    end
+
+    it "does not retry when query execution succeeds" do
+      mixin_settings['connection_retry_attempts'] = 2
+      queue = Queue.new
+      plugin.register
+
+      handler = plugin.instance_variable_get(:@statement_handler)
+      allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_call_original
+      expect(plugin.logger).not_to receive(:error)
+
+      plugin.run(queue)
+      plugin.stop
+    end
   end
 
   context "when encoding of some columns need to be changed" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.4.1
+  version: 5.4.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-04 00:00:00.000000000 Z
+date: 2023-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -333,7 +333,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Integration with JDBC - input and filter plugins