logstash-integration-jdbc 5.4.11 → 5.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7604bb8967c9d63f585d14622ab18b70bc0d9e9b34e43bca6b6c59c9d05a6b76
-  data.tar.gz: 2e9d142376d12bbd1313142e79174d540d53e098e481330262e784c8177e4898
+  metadata.gz: e155ca5542f41d749481a67cf65cf28ad194f019defe6368f765cd1a0ccf3860
+  data.tar.gz: 81cbc828cbbdecb5d3265c07f5777f68c3103252edbc772868c1aa8f1323b566
 SHA512:
-  metadata.gz: 165749e33fed7d98b9635f6a1d74cdb05623563a3942d34c093ef761cd8e4f4784e6eb467a699909e5420ceef8fb4779c00e1b154845838052aa278a592ddc2a
-  data.tar.gz: b9a4e7822ea60c3dde25edd6a1a6e72837069a993046a050a05aa32e92172d4689624df12e85c31b79f62c40fd0f45f98700e07a678a7ea2656d9730f2f14a8e
+  metadata.gz: 0d3f9f07bd92ff1c21f417795e20edbd5f78542181a00f4bfd770dd7f9fb1066df8bbec9af242126b13148049cc70f41190b5246104a23c549af6a62338807c1
+  data.tar.gz: 71c92b8dba18479be7d69f64b580d2bd99902d7d330fd1633997e8134c69403f640e1ba555a1b1640d6737de8b53c68d51e4cab75b4da82eeda7fbf549f72c0c

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.5.0
+  - Feat: add support for SQL `DATE` columns to jdbc static and streaming filters [#171](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/171)
+
 ## 5.4.11
   - Fixes an issue in which any one instance of a JDBC input plugin using `jdbc_default_timezone` changes the behaviour of plugin instances that do _not_ use `jdbc_default_timezone`, ensuring that timezone offsets remain consistent for each instance of the plugin _as configured_ [#151](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/151)
   - Fixes an exception that could occur while reloading `jdbc_static` databases when the underlying connection to the remote has been broken [#165](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/165)

data/lib/logstash/filters/jdbc/lookup.rb

@@ -1,10 +1,12 @@
 # encoding: utf-8
 require_relative "lookup_result"
 require "logstash/util/loggable"
+require "logstash/plugin_mixins/jdbc/value_handler"
 
 module LogStash module Filters module Jdbc
   class Lookup
     include LogStash::Util::Loggable
+    include LogStash::PluginMixins::Jdbc::ValueHandler
 
     class Sprintfier
       def initialize(param)
@@ -134,15 +136,13 @@ module LogStash module Filters module Jdbc
 
     def load_data_from_local(local, query, params, result)
       local.fetch(query, params).each do |row|
-        stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
-        result.push(stringified)
+        result.push(extract_values_from(row))
       end
     end
 
     def load_data_from_prepared(_local, _query, params, result)
       @prepared_statement.call(params).each do |row|
-        stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
-        result.push(stringified)
+        result.push(extract_values_from(row))
       end
     end
 

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -6,6 +6,7 @@ require "date"
 require_relative "value_tracking"
 require_relative "timezone_proxy"
 require_relative "statement_handler"
+require_relative "value_handler"
 
 java_import java.util.concurrent.locks.ReentrantLock
 
@@ -13,6 +14,7 @@ java_import java.util.concurrent.locks.ReentrantLock
 # for potential reuse in other plugins (input/output)
 module LogStash  module PluginMixins module Jdbc
   module Jdbc
+    include LogStash::PluginMixins::Jdbc::ValueHandler
     # This method is called when someone includes this module
     def self.included(base)
       # Add these methods to the 'base' given.
@@ -252,25 +254,6 @@ module LogStash  module PluginMixins module Jdbc
         row[@tracking_column.to_sym]
       end
     end
-
-    private
-    #Stringify row keys and decorate values when necessary
-    def extract_values_from(row)
-      Hash[row.map { |k, v| [k.to_s, decorate_value(v)] }]
-    end
-
-    private
-    def decorate_value(value)
-      case value
-      when Time
-        # transform it to LogStash::Timestamp as required by LS
-        LogStash::Timestamp.new(value)
-      when Date, DateTime
-        LogStash::Timestamp.new(value.to_time)
-      else
-        value
-      end
-    end
   end
 end end end
 

data/lib/logstash/plugin_mixins/jdbc/value_handler.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "time"
+require "date"
+
+module LogStash module PluginMixins module Jdbc
+  # Provides functions to extract the row's values, ensuring column types
+  # are properly decorated to become coercible to a LogStash::Event.
+  module ValueHandler
+    # Stringify the row keys and decorate values when necessary
+    def extract_values_from(row)
+      Hash[row.map { |k, v| [k.to_s, decorate_value(v)] }]
+    end
+
+    # Decorate the value so it can be used as a LogStash::Event field
+    def decorate_value(value)
+      case value
+      when Date, DateTime
+        value.to_time
+      else
+        value
+      end
+    end
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc_streaming/statement_handler.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require "logstash/util/loggable"
+require "logstash/plugin_mixins/jdbc/value_handler"
 
 module LogStash module PluginMixins module JdbcStreaming
   # so as to not clash with the class of the same name and function in the jdbc input
@@ -7,6 +8,8 @@ module LogStash module PluginMixins module JdbcStreaming
   # this duplication can be removed in a universal plugin
 
   class StatementHandler
+    include LogStash::PluginMixins::Jdbc::ValueHandler
+
     def self.build_statement_handler(plugin)
       klass = plugin.use_prepared_statements ? PreparedStatementHandler : NormalStatementHandler
       klass.new(plugin)
@@ -86,7 +89,7 @@ module LogStash module PluginMixins module JdbcStreaming
     def execute_extract_records(db, params, result)
       dataset = db[statement, params] # returns a Sequel dataset
       dataset.all do |row|
-        result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} # Stringify row keys
+        result.push extract_values_from(row)
       end
     end
 
@@ -113,7 +116,7 @@ module LogStash module PluginMixins module JdbcStreaming
     def execute_extract_records(db, params, result)
       records = db.call(name, params) # returns an array of hashes
       records.each do |row|
-        result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
+        result.push extract_values_from(row)
       end
     end
 

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.4.11'
+  s.version         = '5.5.0'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/integration/jdbc_static_spec.rb

@@ -116,6 +116,45 @@ module LogStash module Filters
           plugin.filter(event)
           expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
         end
+
+        context 'and record with temporal columns' do
+          let(:loader_statement) { "SELECT ip, name, location, entry_date, entry_time, timestamp FROM reference_table" }
+          let(:local_db_objects) do
+            [
+              {
+                "name" => "servers",
+                "columns" => [
+                  %w[ip varchar(64)],
+                  %w[name varchar(64)],
+                  %w[location varchar(64)],
+                  %w[entry_date date],
+                  %w[entry_time time],
+                  %w[timestamp timestamp]
+                ]
+              },
+            ]
+          end
+
+          before(:each) { plugin.register }
+
+          subject { event.get("server").first }
+
+          it "maps the DATE to a Logstash Timestamp" do
+            plugin.filter(event)
+            expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+          end
+
+          it "maps the TIME field to a Logstash Timestamp" do
+            plugin.filter(event)
+            now = DateTime.now
+            expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+          end
+
+          it "maps the TIMESTAMP to a Logstash Timestamp" do
+            plugin.filter(event)
+            expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+          end
+        end
       end
 
       context "under normal conditions when index_columns is not specified" do

data/spec/filters/integration/jdbcstreaming_spec.rb

@@ -63,7 +63,32 @@ module LogStash module Filters
       end
     end
 
-    describe "In Prepared Statement mode, found record - uses row" do
+    describe 'found record with temporal columns' do
+      let(:idx) { 200 }
+      let(:statement) { "SELECT entry_date, entry_time, timestamp FROM reference_table WHERE ip = :ip" }
+
+      before(:each) { plugin.register }
+
+      subject { event.get("server").first }
+
+      it "maps the DATE to a Logstash Timestamp" do
+        plugin.filter(event)
+        expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+      end
+
+      it "maps the TIME field to a Logstash Timestamp" do
+        plugin.filter(event)
+        now = DateTime.now
+        expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+      end
+
+      it "maps the TIMESTAMP to a Logstash Timestamp" do
+        plugin.filter(event)
+        expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+      end
+    end
+
+    context 'prepared statement mode' do
       let(:idx) { 200 }
       let(:statement) { "SELECT name, location FROM reference_table WHERE ip = ?" }
       let(:settings) do
@@ -82,10 +107,37 @@ module LogStash module Filters
           "sequel_opts" => {"pool_timeout" => 600}
         }
       end
-      it "fills in the target" do
-        plugin.filter(event)
-        expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
-        expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+
+      describe "found record - uses row" do
+        it "fills in the target" do
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+          expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+        end
+      end
+
+      describe 'found record with temporal columns' do
+        let(:statement) { "SELECT entry_date, entry_time, timestamp FROM reference_table WHERE ip = ?" }
+
+        before(:each) { plugin.register }
+
+        subject { event.get("server").first }
+
+        it "maps the DATE to a Logstash Timestamp" do
+          plugin.filter(event)
+          expect(subject['entry_date']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+        end
+
+        it "maps the TIME field to a Logstash Timestamp" do
+          plugin.filter(event)
+          now = DateTime.now
+          expect(subject['entry_time']).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+        end
+
+        it "maps the TIMESTAMP to a Logstash Timestamp" do
+          plugin.filter(event)
+          expect(subject['timestamp']).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+        end
       end
     end
 

data/spec/inputs/integration/integ_spec.rb

@@ -66,6 +66,29 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         expect(event.get('first_name')).to eq('David')
       end
     end
+
+    context 'with temporal columns' do
+      let(:settings) do
+        super().merge("statement" => 'SELECT ENTRY_DATE, ENTRY_TIME, TIMESTAMP FROM "employee" WHERE EMP_NO = 2')
+      end
+
+      before(:each) { plugin.run(queue) }
+
+      subject(:event) { queue.pop }
+
+      it "maps the DATE to a Logstash Timestamp" do
+        expect(event.get('entry_date')).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1)))
+      end
+
+      it "maps the TIME field to a Logstash Timestamp" do
+        now = DateTime.now
+        expect(event.get('entry_time')).to eq(LogStash::Timestamp.new(Time.new(now.year, now.month, now.day, 10, 5, 0)))
+      end
+
+      it "maps the TIMESTAMP to a Logstash Timestamp" do
+        expect(event.get('timestamp')).to eq(LogStash::Timestamp.new(Time.new(2003, 2, 1, 1, 2, 3)))
+      end
+    end
   end
 
   context "when supplying a non-existent library" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.4.11
+  version: 5.5.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-23 00:00:00.000000000 Z
+date: 2024-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -17,8 +17,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.3'
   name: jar-dependencies
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -34,8 +34,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 6.5.0
   name: logstash-core
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -65,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -79,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.74.0
   name: sequel
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -93,8 +93,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: lru_redux
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -107,8 +107,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: tzinfo
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -121,8 +121,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: tzinfo-data
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -135,8 +135,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
   name: logstash-mixin-ecs_compatibility_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -149,8 +149,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-validator_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -163,8 +163,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-event_support
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -177,8 +177,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-scheduler
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -191,8 +191,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: childprocess
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -205,8 +205,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.3'
   name: logstash-devutils
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -219,8 +219,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: timecop
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -266,6 +266,7 @@ files:
 - lib/logstash/plugin_mixins/jdbc/sequel_bootstrap.rb
 - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
 - lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb
+- lib/logstash/plugin_mixins/jdbc/value_handler.rb
 - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
 - lib/logstash/plugin_mixins/jdbc_streaming.rb
 - lib/logstash/plugin_mixins/jdbc_streaming/cache_payload.rb
@@ -323,7 +324,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Integration with JDBC - input and filter plugins