logstash-integration-jdbc 5.4.10 → 5.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b3c2ba844927de3fe8fd1ea8a23fe09af8da74835eb8c87a83e4de9f67b3d21
-  data.tar.gz: 401acb665aacbf8e9f75b6d5092985998d766217619fcc2d26c7d078f0d4f82f
+  metadata.gz: e155ca5542f41d749481a67cf65cf28ad194f019defe6368f765cd1a0ccf3860
+  data.tar.gz: 81cbc828cbbdecb5d3265c07f5777f68c3103252edbc772868c1aa8f1323b566
 SHA512:
-  metadata.gz: c68b79a3bdb9d0529dbff7b1c0e80dcd0e840d57f6c760c056fbbd5525be295bfedb8ddf7b2450ab125eee14c039cf61a7ea38aca81e597a98ea815b51955bab
-  data.tar.gz: eb872090eb0dfc7bb073aa8c3f992679ecfcb66979c2de6f6838ff17f6acc297d82140cb60ef71a55dc910ad17c9ecab921e62e9d899a4943949ba3003b38a7a
+  metadata.gz: 0d3f9f07bd92ff1c21f417795e20edbd5f78542181a00f4bfd770dd7f9fb1066df8bbec9af242126b13148049cc70f41190b5246104a23c549af6a62338807c1
+  data.tar.gz: 71c92b8dba18479be7d69f64b580d2bd99902d7d330fd1633997e8134c69403f640e1ba555a1b1640d6737de8b53c68d51e4cab75b4da82eeda7fbf549f72c0c

data/CHANGELOG.md

@@ -1,6 +1,13 @@
+## 5.5.0
+  - Feat: add support for SQL `DATE` columns to jdbc static and streaming filters [#171](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/171)
+
+## 5.4.11
+  - Fixes an issue in which any one instance of a JDBC input plugin using `jdbc_default_timezone` changes the behaviour of plugin instances that do _not_ use `jdbc_default_timezone`, ensuring that timezone offsets remain consistent for each instance of the plugin _as configured_ [#151](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/151)
+  - Fixes an exception that could occur while reloading `jdbc_static` databases when the underlying connection to the remote has been broken [#165](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/165)
+
 ## 5.4.10
- - Adds retry mechanism when checkout Derby from SVN repository [#158](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/158)
- - [DOC] add known limitations and settings for connection issue [#167](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/167)
+  - Adds retry mechanism when checkout Derby from SVN repository [#158](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/158)
+  - [DOC] add known limitations and settings for connection issue [#167](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/167)
 
 ## 5.4.9
   - Fix Derby missed driver classes when built locally for version 10.15 [#160](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/160)

data/lib/logstash/filters/jdbc/basic_database.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require "fileutils"
-require "sequel"
+require "logstash/plugin_mixins/jdbc/sequel_bootstrap"
 require "sequel/adapters/jdbc"
 require "java"
 require "logstash/util/loggable"

data/lib/logstash/filters/jdbc/lookup.rb

@@ -1,10 +1,12 @@
 # encoding: utf-8
 require_relative "lookup_result"
 require "logstash/util/loggable"
+require "logstash/plugin_mixins/jdbc/value_handler"
 
 module LogStash module Filters module Jdbc
   class Lookup
     include LogStash::Util::Loggable
+    include LogStash::PluginMixins::Jdbc::ValueHandler
 
     class Sprintfier
       def initialize(param)
@@ -134,15 +136,13 @@ module LogStash module Filters module Jdbc
 
     def load_data_from_local(local, query, params, result)
       local.fetch(query, params).each do |row|
-        stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
-        result.push(stringified)
+        result.push(extract_values_from(row))
       end
     end
 
     def load_data_from_prepared(_local, _query, params, result)
       @prepared_statement.call(params).each do |row|
-        stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
-        result.push(stringified)
+        result.push(extract_values_from(row))
       end
     end
 

data/lib/logstash/filters/jdbc/read_only_database.rb

@@ -12,7 +12,7 @@ module LogStash module Filters module Jdbc
         if connected?
           result = @db[statement].count
         else
-          debug_log_messages.concat("and there is no connection to the remote db at this time")
+          debug_log_messages << "and there is no connection to the remote db at this time"
         end
       rescue ::Sequel::Error => err
         # a fatal issue
@@ -32,7 +32,7 @@ module LogStash module Filters module Jdbc
         if connected?
           result = @db[statement].all
         else
-          debug_log_messages.concat("and there is no connection to the remote db at this time")
+          debug_log_messages << "and there is no connection to the remote db at this time"
         end
       rescue ::Sequel::Error => err
         # a fatal issue

data/lib/logstash/plugin_mixins/jdbc/common.rb

@@ -24,7 +24,8 @@ module LogStash module PluginMixins module Jdbc
       return @driver_impl if @driver_impl ||= nil
 
       require "java"
-      require "sequel"
+
+      require_relative "sequel_bootstrap"
       require "sequel/adapters/jdbc"
 
       # execute all the driver loading related duties in a serial fashion to avoid

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -6,6 +6,7 @@ require "date"
 require_relative "value_tracking"
 require_relative "timezone_proxy"
 require_relative "statement_handler"
+require_relative "value_handler"
 
 java_import java.util.concurrent.locks.ReentrantLock
 
@@ -13,6 +14,7 @@ java_import java.util.concurrent.locks.ReentrantLock
 # for potential reuse in other plugins (input/output)
 module LogStash  module PluginMixins module Jdbc
   module Jdbc
+    include LogStash::PluginMixins::Jdbc::ValueHandler
     # This method is called when someone includes this module
     def self.included(base)
       # Add these methods to the 'base' given.
@@ -252,25 +254,6 @@ module LogStash  module PluginMixins module Jdbc
         row[@tracking_column.to_sym]
       end
     end
-
-    private
-    #Stringify row keys and decorate values when necessary
-    def extract_values_from(row)
-      Hash[row.map { |k, v| [k.to_s, decorate_value(v)] }]
-    end
-
-    private
-    def decorate_value(value)
-      case value
-      when Time
-        # transform it to LogStash::Timestamp as required by LS
-        LogStash::Timestamp.new(value)
-      when Date, DateTime
-        LogStash::Timestamp.new(value.to_time)
-      else
-        value
-      end
-    end
   end
 end end end
 

data/lib/logstash/plugin_mixins/jdbc/sequel_bootstrap.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+require "sequel"
+
+# prevent Sequel's datetime_class from being modified,
+# and ensure behaviour is restored to the library's default
+# if something else in the Ruby VM has already changed it.
+Sequel.synchronize do
+  def Sequel.datetime_class=(klass)
+    # noop
+  end
+  def Sequel.datetime_class
+    ::Time
+  end
+end
+
+# load the named_timezones extension, which will attempt to
+# override the global Sequel::datetime_class; for safety,
+# we reset it once more.
+Sequel.extension(:named_timezones)
+Sequel.datetime_class = ::Time

data/lib/logstash/plugin_mixins/jdbc/value_handler.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "time"
+require "date"
+
+module LogStash module PluginMixins module Jdbc
+  # Provides functions to extract the row's values, ensuring column types
+  # are properly decorated to become coercible to a LogStash::Event.
+  module ValueHandler
+    # Stringify the row keys and decorate values when necessary
+    def extract_values_from(row)
+      Hash[row.map { |k, v| [k.to_s, decorate_value(v)] }]
+    end
+
+    # Decorate the value so it can be used as a LogStash::Event field
+    def decorate_value(value)
+      case value
+      when Date, DateTime
+        value.to_time
+      else
+        value
+      end
+    end
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc_streaming/statement_handler.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/util/loggable"
+require "logstash/plugin_mixins/jdbc/value_handler"
 
 module LogStash module PluginMixins module JdbcStreaming
   # so as to not clash with the class of the same name and function in the jdbc input
@@ -7,6 +8,8 @@ module LogStash module PluginMixins module JdbcStreaming
   # this duplication can be removed in a universal plugin
 
   class StatementHandler
+    include LogStash::PluginMixins::Jdbc::ValueHandler
+
     def self.build_statement_handler(plugin)
       klass = plugin.use_prepared_statements ? PreparedStatementHandler : NormalStatementHandler
       klass.new(plugin)
@@ -86,7 +89,7 @@ module LogStash module PluginMixins module JdbcStreaming
     def execute_extract_records(db, params, result)
       dataset = db[statement, params] # returns a Sequel dataset
       dataset.all do |row|
-        result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} # Stringify row keys
+        result.push extract_values_from(row)
       end
     end
 
@@ -113,7 +116,7 @@ module LogStash module PluginMixins module JdbcStreaming
     def execute_extract_records(db, params, result)
       records = db.call(name, params) # returns an array of hashes
       records.each do |row|
-        result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
+        result.push extract_values_from(row)
       end
     end
 

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.4.10'
+  s.version         = '5.5.0'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/integration/jdbc_static_spec.rb

@@ -116,6 +116,45 @@ module LogStash module Filters
           plugin.filter(event)
           expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
         end
+
+        context 'and record with temporal columns' do
+          let(:loader_statement) { "SELECT ip, name, location, entry_date, entry_time, timestamp FROM reference_table" }
+          let(:local_db_objects) do
+            [
+              {
+                "name" => "servers",
+                "columns" => [
+                  %w[ip varchar(64)],
+                  %w[name varchar(64)],
+                  %w[location varchar(64)],
+                  %w[entry_date date],
+                  %w[entry_time time],
+                  %w[timestamp timestamp]
+                ]
+              },
+            ]
+          end
+
+          before(:each) { plugin.register }
+
+          subject { event.get("server").first }
+
+          it "maps the DATE to a Logstash Timestamp" do
+            plugin.filter(event)
+            expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+          end
+
+          it "maps the TIME field to a Logstash Timestamp" do
+            plugin.filter(event)
+            now = DateTime.now
+            expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+          end
+
+          it "maps the TIMESTAMP to a Logstash Timestamp" do
+            plugin.filter(event)
+            expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+          end
+        end
       end
 
       context "under normal conditions when index_columns is not specified" do

data/spec/filters/integration/jdbcstreaming_spec.rb

@@ -63,7 +63,32 @@ module LogStash module Filters
       end
     end
 
-    describe "In Prepared Statement mode, found record - uses row" do
+    describe 'found record with temporal columns' do
+      let(:idx) { 200 }
+      let(:statement) { "SELECT entry_date, entry_time, timestamp FROM reference_table WHERE ip = :ip" }
+
+      before(:each) { plugin.register }
+
+      subject { event.get("server").first }
+
+      it "maps the DATE to a Logstash Timestamp" do
+        plugin.filter(event)
+        expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+      end
+
+      it "maps the TIME field to a Logstash Timestamp" do
+        plugin.filter(event)
+        now = DateTime.now
+        expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+      end
+
+      it "maps the TIMESTAMP to a Logstash Timestamp" do
+        plugin.filter(event)
+        expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+      end
+    end
+
+    context 'prepared statement mode' do
       let(:idx) { 200 }
       let(:statement) { "SELECT name, location FROM reference_table WHERE ip = ?" }
       let(:settings) do
@@ -82,10 +107,37 @@ module LogStash module Filters
           "sequel_opts" => {"pool_timeout" => 600}
         }
       end
-      it "fills in the target" do
-        plugin.filter(event)
-        expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
-        expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+
+      describe "found record - uses row" do
+        it "fills in the target" do
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+          expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+        end
+      end
+
+      describe 'found record with temporal columns' do
+        let(:statement) { "SELECT entry_date, entry_time, timestamp FROM reference_table WHERE ip = ?" }
+
+        before(:each) { plugin.register }
+
+        subject { event.get("server").first }
+
+        it "maps the DATE to a Logstash Timestamp" do
+          plugin.filter(event)
+          expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+        end
+
+        it "maps the TIME field to a Logstash Timestamp" do
+          plugin.filter(event)
+          now = DateTime.now
+          expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+        end
+
+        it "maps the TIMESTAMP to a Logstash Timestamp" do
+          plugin.filter(event)
+          expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+        end
       end
     end
 

data/spec/inputs/integration/integ_spec.rb

@@ -66,6 +66,29 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         expect(event.get('first_name')).to eq('David')
       end
     end
+
+    context 'with temporal columns' do
+      let(:settings) do
+        super().merge("statement" => 'SELECT ENTRY_DATE, ENTRY_TIME, TIMESTAMP FROM "employee" WHERE EMP_NO = 2')
+      end
+
+      before(:each) { plugin.run(queue) }
+
+      subject(:event) { queue.pop }
+
+      it "maps the DATE to a Logstash Timestamp" do
+        expect(event.get('entry_date')).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+      end
+
+      it "maps the TIME field to a Logstash Timestamp" do
+        now = DateTime.now
+        expect(event.get('entry_time')).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+      end
+
+      it "maps the TIMESTAMP to a Logstash Timestamp" do
+        expect(event.get('timestamp')).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+      end
+    end
   end
 
   context "when supplying a non-existent library" do

data/spec/inputs/jdbc_spec.rb

@@ -601,6 +601,33 @@ describe LogStash::Inputs::Jdbc do
       # With no timezone set, no change should occur
       expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T12:00:00Z"))
     end
+
+    %w(
+        Etc/UTC
+        America/Los_Angeles
+        Europe/Berlin
+        Asia/Tokyo
+      ).each do |local_timezone|
+      context "when host machine has timezone `#{local_timezone}`" do
+        around(:each) do |example|
+          begin
+            previous_tz = ENV['TZ']
+            ENV['TZ'] = local_timezone
+            example.call
+          ensure
+            ENV['TZ'] = previous_tz
+          end
+        end
+
+        let(:tz) { TZInfo::Timezone.get(local_timezone) }
+
+        it "converts the time using the machine's local timezone" do
+          plugin.run(queue)
+          event = queue.pop
+          expect(event.get("custom_time").time).to eq(Time.new(2015,1,1,12,0,0,tz))
+        end
+      end
+    end
   end
 
   context "when iteratively running plugin#run" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.4.10
+  version: 5.5.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-08 00:00:00.000000000 Z
+date: 2024-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -263,8 +263,10 @@ files:
 - lib/logstash/inputs/tzinfo_jruby_patch.rb
 - lib/logstash/plugin_mixins/jdbc/common.rb
 - lib/logstash/plugin_mixins/jdbc/jdbc.rb
+- lib/logstash/plugin_mixins/jdbc/sequel_bootstrap.rb
 - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
 - lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb
+- lib/logstash/plugin_mixins/jdbc/value_handler.rb
 - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
 - lib/logstash/plugin_mixins/jdbc_streaming.rb
 - lib/logstash/plugin_mixins/jdbc_streaming/cache_payload.rb