logstash-integration-jdbc 5.4.11 → 5.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7604bb8967c9d63f585d14622ab18b70bc0d9e9b34e43bca6b6c59c9d05a6b76
-  data.tar.gz: 2e9d142376d12bbd1313142e79174d540d53e098e481330262e784c8177e4898
+  metadata.gz: 4ba7e1b0f0a043e0b58ff727a16decd9f5fb63ac5cc68a1c935af2d59c0235a3
+  data.tar.gz: fa9178574ca2acce0b1f8f0bab0823649e202af2d3911985fb9368a8c3f60c11
 SHA512:
-  metadata.gz: 165749e33fed7d98b9635f6a1d74cdb05623563a3942d34c093ef761cd8e4f4784e6eb467a699909e5420ceef8fb4779c00e1b154845838052aa278a592ddc2a
-  data.tar.gz: b9a4e7822ea60c3dde25edd6a1a6e72837069a993046a050a05aa32e92172d4689624df12e85c31b79f62c40fd0f45f98700e07a678a7ea2656d9730f2f14a8e
+  metadata.gz: 681d44e0f888cd9c7e87cd33551aa62c5aba9524e714af9db334286b41947c0683dcda15b3fb0e8a8b903976599e261dd0c805666db977ed16f6a145f966c507
+  data.tar.gz: d7fb56739d0e2db3751b2956f0e47dfc585173ff7703cb5b9c0d52b438b10358a6678c72267fa3103cd92ef6137da5376c7398bbe89289a665b7a224a2e9a2d8

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.5.1
+  - Document `statement_retry_attempts` and `statement_retry_attempts_wait_time` options
+
+## 5.5.0
+  - Feat: add support for SQL `DATE` columns to jdbc static and streaming filters [#171](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/171)
+
 ## 5.4.11
   - Fixes an issue in which any one instance of a JDBC input plugin using `jdbc_default_timezone` changes the behaviour of plugin instances that do _not_ use `jdbc_default_timezone`, ensuring that timezone offsets remain consistent for each instance of the plugin _as configured_ [#151](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/151)
   - Fixes an exception that could occur while reloading `jdbc_static` databases when the underlying connection to the remote has been broken [#165](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/165)

data/docs/input-jdbc.asciidoc

@@ -238,6 +238,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-sql_log_level>> |<<string,string>>, one of `["fatal", "error", "warn", "info", "debug"]`|No
 | <<plugins-{type}s-{plugin}-statement>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-statement_filepath>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-statement_retry_attempts>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-statement_retry_attempts_wait_time>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
 | <<plugins-{type}s-{plugin}-tracking_column>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-tracking_column_type>> |<<string,string>>, one of `["numeric", "timestamp"]`|No
@@ -623,6 +625,22 @@ with the `parameters` setting.
 
 Path of file containing statement to execute
 
+[id="plugins-{type}s-{plugin}-statement_retry_attempts"]
+===== `statement_retry_attempts`
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+Maximum number of times to try executing a statement.
+
+[id="plugins-{type}s-{plugin}-statement_retry_attempts_wait_time"]
+===== `statement_retry_attempts_wait_time`
+
+  * Value type is <<number,number>>
+  * Default value is `0.5`
+
+Number of seconds to sleep between statement execution attempts.
+
 [id="plugins-{type}s-{plugin}-target"]
 ===== `target`
 

data/lib/logstash/filters/jdbc/lookup.rb

@@ -1,10 +1,12 @@
 # encoding: utf-8
 require_relative "lookup_result"
 require "logstash/util/loggable"
+require "logstash/plugin_mixins/jdbc/value_handler"
 
 module LogStash module Filters module Jdbc
   class Lookup
     include LogStash::Util::Loggable
+    include LogStash::PluginMixins::Jdbc::ValueHandler
 
     class Sprintfier
       def initialize(param)
@@ -134,15 +136,13 @@ module LogStash module Filters module Jdbc
 
     def load_data_from_local(local, query, params, result)
       local.fetch(query, params).each do |row|
-        stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
-        result.push(stringified)
+        result.push(extract_values_from(row))
       end
     end
 
     def load_data_from_prepared(_local, _query, params, result)
       @prepared_statement.call(params).each do |row|
-        stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
-        result.push(stringified)
+        result.push(extract_values_from(row))
       end
     end
 

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -6,6 +6,7 @@ require "date"
 require_relative "value_tracking"
 require_relative "timezone_proxy"
 require_relative "statement_handler"
+require_relative "value_handler"
 
 java_import java.util.concurrent.locks.ReentrantLock
 
@@ -13,6 +14,7 @@ java_import java.util.concurrent.locks.ReentrantLock
 # for potential reuse in other plugins (input/output)
 module LogStash  module PluginMixins module Jdbc
   module Jdbc
+    include LogStash::PluginMixins::Jdbc::ValueHandler
     # This method is called when someone includes this module
     def self.included(base)
       # Add these methods to the 'base' given.
@@ -252,25 +254,6 @@ module LogStash  module PluginMixins module Jdbc
         row[@tracking_column.to_sym]
       end
     end
-
-    private
-    #Stringify row keys and decorate values when necessary
-    def extract_values_from(row)
-      Hash[row.map { |k, v| [k.to_s, decorate_value(v)] }]
-    end
-
-    private
-    def decorate_value(value)
-      case value
-      when Time
-        # transform it to LogStash::Timestamp as required by LS
-        LogStash::Timestamp.new(value)
-      when Date, DateTime
-        LogStash::Timestamp.new(value.to_time)
-      else
-        value
-      end
-    end
   end
 end end end
 

data/lib/logstash/plugin_mixins/jdbc/value_handler.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "time"
+require "date"
+
+module LogStash module PluginMixins module Jdbc
+  # Provides functions to extract the row's values, ensuring column types
+  # are properly decorated to become coercible to a LogStash::Event.
+  module ValueHandler
+    # Stringify the row keys and decorate values when necessary
+    def extract_values_from(row)
+      Hash[row.map { |k, v| [k.to_s, decorate_value(v)] }]
+    end
+
+    # Decorate the value so it can be used as a LogStash::Event field
+    def decorate_value(value)
+      case value
+      when Date, DateTime
+        value.to_time
+      else
+        value
+      end
+    end
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc_streaming/statement_handler.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/util/loggable"
+require "logstash/plugin_mixins/jdbc/value_handler"
 
 module LogStash module PluginMixins module JdbcStreaming
   # so as to not clash with the class of the same name and function in the jdbc input
@@ -7,6 +8,8 @@ module LogStash module PluginMixins module JdbcStreaming
   # this duplication can be removed in a universal plugin
 
   class StatementHandler
+    include LogStash::PluginMixins::Jdbc::ValueHandler
+
     def self.build_statement_handler(plugin)
       klass = plugin.use_prepared_statements ? PreparedStatementHandler : NormalStatementHandler
       klass.new(plugin)
@@ -86,7 +89,7 @@ module LogStash module PluginMixins module JdbcStreaming
     def execute_extract_records(db, params, result)
       dataset = db[statement, params] # returns a Sequel dataset
       dataset.all do |row|
-        result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} # Stringify row keys
+        result.push extract_values_from(row)
       end
     end
 
@@ -113,7 +116,7 @@ module LogStash module PluginMixins module JdbcStreaming
     def execute_extract_records(db, params, result)
       records = db.call(name, params) # returns an array of hashes
       records.each do |row|
-        result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
+        result.push extract_values_from(row)
       end
     end
 

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.4.11'
+  s.version         = '5.5.1'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,7 +23,6 @@ Gem::Specification.new do |s|
   }
 
   # Gem dependencies
-  s.add_development_dependency 'jar-dependencies', '~> 0.3'
 
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   # Restrict use of this plugin to versions of Logstash where support for integration plugins is present.

data/spec/filters/integration/jdbc_static_spec.rb

@@ -116,6 +116,45 @@ module LogStash module Filters
           plugin.filter(event)
           expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
         end
+
+        context 'and record with temporal columns' do
+          let(:loader_statement) { "SELECT ip, name, location, entry_date, entry_time, timestamp FROM reference_table" }
+          let(:local_db_objects) do
+            [
+              {
+                "name" => "servers",
+                "columns" => [
+                  %w[ip varchar(64)],
+                  %w[name varchar(64)],
+                  %w[location varchar(64)],
+                  %w[entry_date date],
+                  %w[entry_time time],
+                  %w[timestamp timestamp]
+                ]
+              },
+            ]
+          end
+
+          before(:each) { plugin.register }
+
+          subject { event.get("server").first }
+
+          it "maps the DATE to a Logstash Timestamp" do
+            plugin.filter(event)
+            expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+          end
+
+          it "maps the TIME field to a Logstash Timestamp" do
+            plugin.filter(event)
+            now = DateTime.now
+            expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+          end
+
+          it "maps the TIMESTAMP to a Logstash Timestamp" do
+            plugin.filter(event)
+            expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+          end
+        end
       end
 
       context "under normal conditions when index_columns is not specified" do

data/spec/filters/integration/jdbcstreaming_spec.rb

@@ -63,7 +63,32 @@ module LogStash module Filters
       end
     end
 
-    describe "In Prepared Statement mode, found record - uses row" do
+    describe 'found record with temporal columns' do
+      let(:idx) { 200 }
+      let(:statement) { "SELECT entry_date, entry_time, timestamp FROM reference_table WHERE ip = :ip" }
+
+      before(:each) { plugin.register }
+
+      subject { event.get("server").first }
+
+      it "maps the DATE to a Logstash Timestamp" do
+        plugin.filter(event)
+        expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+      end
+
+      it "maps the TIME field to a Logstash Timestamp" do
+        plugin.filter(event)
+        now = DateTime.now
+        expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+      end
+
+      it "maps the TIMESTAMP to a Logstash Timestamp" do
+        plugin.filter(event)
+        expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+      end
+    end
+
+    context 'prepared statement mode' do
       let(:idx) { 200 }
       let(:statement) { "SELECT name, location FROM reference_table WHERE ip = ?" }
       let(:settings) do
@@ -82,10 +107,37 @@ module LogStash module Filters
           "sequel_opts" => {"pool_timeout" => 600}
         }
       end
-      it "fills in the target" do
-        plugin.filter(event)
-        expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
-        expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+
+      describe "found record - uses row" do
+        it "fills in the target" do
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+          expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+        end
+      end
+
+      describe 'found record with temporal columns' do
+        let(:statement) { "SELECT entry_date, entry_time, timestamp FROM reference_table WHERE ip = ?" }
+
+        before(:each) { plugin.register }
+
+        subject { event.get("server").first }
+
+        it "maps the DATE to a Logstash Timestamp" do
+          plugin.filter(event)
+          expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+        end
+
+        it "maps the TIME field to a Logstash Timestamp" do
+          plugin.filter(event)
+          now = DateTime.now
+          expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+        end
+
+        it "maps the TIMESTAMP to a Logstash Timestamp" do
+          plugin.filter(event)
+          expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+        end
       end
     end
 

data/spec/inputs/integration/integ_spec.rb

@@ -66,6 +66,29 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         expect(event.get('first_name')).to eq('David')
       end
     end
+
+    context 'with temporal columns' do
+      let(:settings) do
+        super().merge("statement" => 'SELECT ENTRY_DATE, ENTRY_TIME, TIMESTAMP FROM "employee" WHERE EMP_NO = 2')
+      end
+
+      before(:each) { plugin.run(queue) }
+
+      subject(:event) { queue.pop }
+
+      it "maps the DATE to a Logstash Timestamp" do
+        expect(event.get('entry_date')).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+      end
+
+      it "maps the TIME field to a Logstash Timestamp" do
+        now = DateTime.now
+        expect(event.get('entry_time')).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+      end
+
+      it "maps the TIMESTAMP to a Logstash Timestamp" do
+        expect(event.get('timestamp')).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+      end
+    end
   end
 
   context "when supplying a non-existent library" do

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.4.11
+  version: 5.5.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-23 00:00:00.000000000 Z
+date: 2024-12-03 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
-  name: jar-dependencies
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -266,6 +252,7 @@ files:
 - lib/logstash/plugin_mixins/jdbc/sequel_bootstrap.rb
 - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
 - lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb
+- lib/logstash/plugin_mixins/jdbc/value_handler.rb
 - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
 - lib/logstash/plugin_mixins/jdbc_streaming.rb
 - lib/logstash/plugin_mixins/jdbc_streaming/cache_payload.rb