logstash-integration-jdbc 5.3.0 → 5.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9937a8c1c5ab26acb4cd887f4bdb9dfff51e9348c7187ef2d44ff6055d137ee
-  data.tar.gz: 0042c3e29b5b08abe8baa4f84e4c240960bbc615778df9c77415f8db7985c831
+  metadata.gz: 71bdfca5d1e5e86b7d86b5c49caf237cfc882e31dab78d63e169241e1c96814d
+  data.tar.gz: 5c07e44042c4a6e6e29b8ecc7fb7631147100a27777c6fdd2a22015ac7fd3b4e
 SHA512:
-  metadata.gz: dbc38871c106b045a500645cf6c4517b30ce15e52bfc68520845e22276efdb4a21df3e4a7709ae682cd6b55ff76e063a615b9fe397795272535aa7c5298a9a92
-  data.tar.gz: 70b8e1dc84ba60d478affb22c93400cb153d4d0adc4904667970220ac03cb9d22b94e28f1b1a5504a30b8592c543d3234ee441d0ed57a31aefef70dfb6c2b9a3
+  metadata.gz: 53e45e47383b15f559e58969e96c0217c43ceb5ba3528db5d6ce3bad10628da6e4c14442aeb5a8795e17b667ac558e5b1377134bb0ecc5e60a375e6ebe106b2c
+  data.tar.gz: d33a2cdea75e56fbe338e175276679e2910b4023c956bc8df7d0795f58140b39da1b9167d668c7665e27b2577c086409b3e390b2a3f8d86c4f9a3e3cdef343e3

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 5.4.1
+  - Bugfix leak which happened in creating a new Database pool for every query. The pool is now crated on registration and closed on plugin's `stop` [#119](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/119)
+ 
+## 5.4.0
+  - Ambiguous Timestamp Support [#92](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/92)
+    - FIX: when encountering an ambiguous timestamp, the JDBC Input no longer crashes
+    - Added support for disambiguating timestamps in daylight saving time (DST) overlap periods
+
 ## 5.3.0
   - Refactor: start using scheduler mixin [#110](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/110)
 

data/docs/filter-jdbc_streaming.asciidoc

@@ -21,7 +21,7 @@ include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
-This filter executes a SQL query and store the result set in the field
+This filter executes a SQL query and stores the result set in the field
 specified as `target`.
 It will cache the results locally in an LRU cache with expiry.
 

data/docs/input-jdbc.asciidoc

@@ -286,10 +286,14 @@ JDBC connection string
 ===== `jdbc_default_timezone`
 
   * Value type is <<string,string>>
+  ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3`
+  ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]`
   * There is no default value for this setting.
 
-Timezone conversion.
-Logstash (and Elasticsearch) expects that timestamps are expressed in UTC terms.
+[id="plugins-{type}s-{plugin}-jdbc_timezone_conv"]
+====== Timezone conversion
+
+Logstash and Elasticsearch expect timestamps to be expressed in UTC terms.
 If your database has recorded timestamps that are relative to another timezone,
 the database timezone if you will, then set this setting to be the timezone that
 the database is using. However, as SQL does not allow for timezone data in
@@ -299,7 +303,15 @@ in relative UTC time in ISO8601 format.
 
 Using this setting will manually assign a specified timezone offset, instead
 of using the timezone setting of the local machine.  You must use a canonical
-timezone, *America/Denver*, for example.
+timezone, `America/Denver`, for example.
+
+[id="plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"]
+===== Ambiguous timestamps
+
+While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline.
+For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous.
+
+When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`).
 
 [id="plugins-{type}s-{plugin}-plugin_timezone"]
 ===== `plugin_timezone`

data/lib/logstash/inputs/jdbc.rb

@@ -261,8 +261,6 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
       end
     end
 
-    prepare_jdbc_connection
-
     if @use_column_value
       # Raise an error if @use_column_value is true, but no @tracking_column is set
       if @tracking_column.nil?
@@ -305,6 +303,20 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
         converters[encoding] = converter
       end
     end
+
+    load_driver
+    begin
+      open_jdbc_connection
+    rescue Sequel::DatabaseConnectionError,
+           Sequel::DatabaseError,
+           Sequel::InvalidValue,
+           Java::JavaSql::SQLException => e
+      details = { exception: e.class, message: e.message }
+      details[:cause] = e.cause.inspect if e.cause
+      details[:backtrace] = e.backtrace if @logger.debug?
+      @logger.warn("Exception when executing JDBC query", details)
+      raise(LogStash::ConfigurationError, "Can't create a connection pool to the database")
+    end
   end # def register
 
   # test injection points
@@ -317,7 +329,6 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   end
 
   def run(queue)
-    load_driver
     if @schedule
       # scheduler input thread name example: "[my-oracle]|input|jdbc|scheduler"
       scheduler.cron(@schedule) { execute_query(queue) }

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -4,6 +4,7 @@ require "logstash/config/mixin"
 require "time"
 require "date"
 require_relative "value_tracking"
+require_relative "timezone_proxy"
 require_relative "statement_handler"
 
 java_import java.util.concurrent.locks.ReentrantLock
@@ -82,7 +83,8 @@ module LogStash  module PluginMixins module Jdbc
       # Using this setting will manually assign a specified timezone offset, instead
       # of using the timezone setting of the local machine.  You must use a canonical
       # timezone, *America/Denver*, for example.
-      config :jdbc_default_timezone, :validate => :string
+      config :jdbc_default_timezone, :validate => :jdbc_timezone_spec
+      extend TimezoneProxy::JDBCTimezoneSpecValidator
 
       # General/Vendor-specific Sequel configuration options.
       #
@@ -157,7 +159,7 @@ module LogStash  module PluginMixins module Jdbc
       @database.extension(:pagination)
       if @jdbc_default_timezone
         @database.extension(:named_timezones)
-        @database.timezone = @jdbc_default_timezone
+        @database.timezone = TimezoneProxy.load(@jdbc_default_timezone)
       end
       if @jdbc_validate_connection
         @database.extension(:connection_validator)
@@ -186,22 +188,14 @@ module LogStash  module PluginMixins module Jdbc
       end
     end
 
-    public
-    def prepare_jdbc_connection
-      @connection_lock = ReentrantLock.new
-    end
-
     public
     def close_jdbc_connection
       begin
         # pipeline restarts can also close the jdbc connection, block until the current executing statement is finished to avoid leaking connections
         # connections in use won't really get closed
-        @connection_lock.lock
         @database.disconnect if @database
       rescue => e
         @logger.warn("Failed to close connection", :exception => e)
-      ensure
-        @connection_lock.unlock
       end
     end
 
@@ -209,8 +203,6 @@ module LogStash  module PluginMixins module Jdbc
     def execute_statement
       success = false
       begin
-        @connection_lock.lock
-        open_jdbc_connection
         sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
         @tracking_column_warning_sent = false
         @statement_handler.perform_query(@database, @value_tracker.value) do |row|
@@ -218,16 +210,16 @@ module LogStash  module PluginMixins module Jdbc
           yield extract_values_from(row)
         end
         success = true
-      rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError, Java::JavaSql::SQLException => e
+      rescue Sequel::DatabaseConnectionError,
+             Sequel::DatabaseError,
+             Sequel::InvalidValue,
+             Java::JavaSql::SQLException => e
         details = { exception: e.class, message: e.message }
         details[:cause] = e.cause.inspect if e.cause
         details[:backtrace] = e.backtrace if @logger.debug?
         @logger.warn("Exception when executing JDBC query", details)
       else
         @value_tracker.set_value(sql_last_value)
-      ensure
-        close_jdbc_connection
-        @connection_lock.unlock
       end
       return success
     end

data/lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+require 'tzinfo'
+
+module LogStash module PluginMixins module Jdbc
+  ##
+  # This `TimezoneProxy` allows timezone specs to include extensions indicating preference for ambiguous handling.
+  # @see TimezoneProxy::parse
+  module TimezoneProxy
+    ##
+    # @param timezone_spec [String]: a timezone spec, consisting of any valid timezone identifier
+    #                                followed by square-bracketed extensions. Currently-supported
+    #                                extensions are:
+    #                                `dst_enabled_on_overlap:(true|false)`: when encountering an ambiguous time
+    #                                                                       due to daylight-savings transition,
+    #                                                                       assume DST to be either enabled or
+    #                                                                       disabled instead of raising an
+    #                                                                       AmbiguousTime exception
+    # @return [TZInfo::Timezone]
+    def self.load(timezone_spec)
+      # re-load pass-through
+      return timezone_spec if timezone_spec.kind_of?(::TZInfo::Timezone)
+
+      parsed_spec = /\A(?<name>[^\[]+)(\[(?<extensions>[^\]]*)\])?\z/.match(timezone_spec)
+
+      timezone = ::TZInfo::Timezone.get(parsed_spec[:name])
+      return timezone unless parsed_spec[:extensions]
+
+      parsed_spec[:extensions].split(';').each do |extension_spec|
+        timezone = case extension_spec
+                   when 'dst_enabled_on_overlap:true'  then timezone.dup.extend(PeriodForLocalWithDSTPreference::ON)
+                   when 'dst_enabled_on_overlap:false' then timezone.dup.extend(PeriodForLocalWithDSTPreference::OFF)
+                   else fail(ArgumentError, "Invalid timezone extension `#{extension_spec}`")
+                   end
+      end
+
+      timezone
+    end
+
+    module JDBCTimezoneSpecValidator
+      def validate_value(value, validator_name)
+        return super(value, validator_name) unless validator_name == :jdbc_timezone_spec
+
+        [true, TimezoneProxy.load(value)] rescue [false, $!.message]
+      end
+    end
+
+    ##
+    # @api private
+    class PeriodForLocalWithDSTPreference < Module
+      def initialize(default_dst_enabled_on_overlap)
+        define_method(:period_for_local) do |localtime, dst_enabled_on_overlap=nil, &dismabiguation_block|
+          super(localtime, dst_enabled_on_overlap.nil? ? default_dst_enabled_on_overlap : dst_enabled_on_overlap, &dismabiguation_block)
+        end
+      end
+
+      ON = new(true)
+      OFF = new(false)
+    end
+  end
+end; end; end

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -14,7 +14,7 @@ module LogStash module PluginMixins module Jdbc
         # use this irrespective of the jdbc_default_timezone setting
         NumericValueTracker.new(handler)
       else
-        if plugin.jdbc_default_timezone.nil? || plugin.jdbc_default_timezone.empty?
+        if plugin.jdbc_default_timezone.nil?
           # no TZ stuff for Sequel, use Time
           TimeValueTracker.new(handler)
         else

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.3.0'
+  s.version         = '5.4.1'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/integration/integ_spec.rb

@@ -76,10 +76,8 @@ describe LogStash::Inputs::Jdbc, :integration => true do
     end
 
     it "should not register correctly" do
-      plugin.register
-      q = Queue.new
       expect do
-        plugin.run(q)
+        plugin.register
       end.to raise_error(::LogStash::PluginLoadingError)
     end
   end
@@ -92,16 +90,13 @@ describe LogStash::Inputs::Jdbc, :integration => true do
     end
 
     it "log warning msg when plugin run" do
-      plugin.register
       expect( plugin ).to receive(:log_java_exception)
       expect(plugin.logger).to receive(:warn).once.with("Exception when executing JDBC query",
                                                         hash_including(:message => instance_of(String)))
-      q = Queue.new
-      expect{ plugin.run(q) }.not_to raise_error
+      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
     end
 
     it "should log (native) Java driver error" do
-      plugin.register
       expect( org.apache.logging.log4j.LogManager ).to receive(:getLogger).and_wrap_original do |m, *args|
         logger = m.call(*args)
         expect( logger ).to receive(:error) do |_, e|
@@ -109,8 +104,7 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         end.and_call_original
         logger
       end
-      q = Queue.new
-      expect{ plugin.run(q) }.not_to raise_error
+      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
     end
   end
 end

data/spec/inputs/jdbc_spec.rb

@@ -696,7 +696,7 @@ describe LogStash::Inputs::Jdbc do
         "last_run_metadata_path" => Stud::Temporary.pathname }
     end
 
-    let(:nums) { [BigDecimal.new(10), BigDecimal.new(20), BigDecimal.new(30), BigDecimal.new(40), BigDecimal.new(50)] }
+    let(:nums) { [BigDecimal(10), BigDecimal(20), BigDecimal(30), BigDecimal(40), BigDecimal(50)] }
 
     before do
       plugin.register
@@ -1504,6 +1504,49 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "when retrieving records with ambiguous timestamps" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from types_table",
+        "jdbc_default_timezone" => jdbc_default_timezone
+      }
+    end
+
+    before(:each) do
+      db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '2021-11-07 01:23:45', 95.67)"
+      plugin.register
+    end
+
+    context "when initialized with a preference for DST being enabled" do
+      let(:jdbc_default_timezone) { 'America/Chicago[dst_enabled_on_overlap:true]' }
+
+      it 'treats the timestamp column as if DST was enabled' do
+        plugin.run(queue)
+        event = queue.pop
+        expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("2021-11-07T06:23:45Z")
+      end
+    end
+    context "when initialized with a preference for DST being disabled" do
+      let(:jdbc_default_timezone) { 'America/Chicago[dst_enabled_on_overlap:false]' }
+
+      it 'treats the timestamp column as if DST was disabled' do
+        plugin.run(queue)
+        event = queue.pop
+        expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("2021-11-07T07:23:45Z")
+      end
+    end
+    context "when initialized without a preference for DST being enabled or disabled" do
+      before(:each) { allow(plugin.logger).to receive(:warn) }
+      let(:jdbc_default_timezone) { 'America/Chicago' }
+
+      it 'the error results in helpful log warning' do
+        plugin.run(queue)
+        expect(plugin.logger).to have_received(:warn).with(a_string_including("Exception when executing JDBC query"), a_hash_including(:message => a_string_including("2021-11-07 01:23:45 is an ambiguous local time")))
+      end
+    end
+  end
+
   context "when an unreadable jdbc_driver_path entry is present" do
     let(:driver_jar_path) do
       jar_file = $CLASSPATH.find { |name| name.index(Jdbc::Derby.driver_jar) }
@@ -1522,16 +1565,12 @@ describe LogStash::Inputs::Jdbc do
       { "statement" => "SELECT * from types_table", "jdbc_driver_library" => invalid_driver_jar_path }
     end
 
-    before do
-      plugin.register
-    end
-
     after do
       plugin.stop
     end
 
     it "raise a loading error" do
-      expect { plugin.run(queue) }.
+      expect { plugin.register }.
           to raise_error(LogStash::PluginLoadingError, /unable to load .*? from :jdbc_driver_library, file not readable/)
     end
   end

data/spec/plugin_mixins/jdbc/timezone_proxy_spec.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin_mixins/jdbc/timezone_proxy"
+
+describe LogStash::PluginMixins::Jdbc::TimezoneProxy do
+  subject(:timezone) { described_class.load(timezone_spec) }
+
+  context 'when handling a daylight-savings ambiguous time' do
+    context 'without extensions' do
+      let(:timezone_spec) { 'America/Los_Angeles[]' }
+      it 'raises an AmbiguousTime error' do
+        expect { timezone.local_time(2021,11,7,1,17) }.to raise_error(::TZInfo::AmbiguousTime)
+      end
+    end
+    context 'with extension `dst_enabled_on_overlap:true`' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:true]' }
+      it 'resolves as if DST were enabled' do
+        timestamp = timezone.local_time(2021,11,7,1,17)
+        aggregate_failures do
+          expect(timestamp.dst?).to be true
+          expect(timestamp.zone).to eq('PDT') # Pacific Daylight Time
+          expect(timestamp.getutc).to eq(Time.utc(2021,11,7,8,17))
+          expect(timestamp.utc_offset).to eq( -7 * 3600 )
+        end
+      end
+      end
+    context 'with extension `dst_enabled_on_overlap:false`' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:false]' }
+      it 'resolves as if DST were disabled' do
+        timestamp = timezone.local_time(2021,11,7,1,17)
+        aggregate_failures do
+          expect(timestamp.dst?).to be false
+          expect(timestamp.zone).to eq('PST') # Pacific Standard Time
+          expect(timestamp.getutc).to eq(Time.utc(2021,11,7,9,17))
+          expect(timestamp.utc_offset).to eq( -8 * 3600 )
+        end
+      end
+    end
+  end
+
+  context '#load' do
+    context 'when spec is a normal timezone instance' do
+      let(:timezone_spec) { ::TZInfo::Timezone.get('America/Los_Angeles') }
+      it 'returns that instance' do
+        expect(timezone).to be(timezone_spec)
+      end
+    end
+    context 'when spec is a valid unextended timezone spec' do
+      let(:timezone_spec) { 'America/Los_Angeles' }
+      it 'returns the canonical timezone' do
+        expect(timezone).to eq(::TZInfo::Timezone.get('America/Los_Angeles'))
+      end
+    end
+    context 'when spec is an invalid timezone spec' do
+      let(:timezone_spec) { 'NotAValidTimezoneIdentifier' }
+
+      it 'propagates the TZInfo exception' do
+        expect { timezone }.to raise_exception(::TZInfo::InvalidTimezoneIdentifier)
+      end
+    end
+    context 'with invalid extension' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:false;nope:wrong]' }
+      it 'raises an exception with a helpful message' do
+        expect { timezone }.to raise_exception(ArgumentError, a_string_including("Invalid timezone extension `nope:wrong`"))
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.3.0
+  version: 5.4.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-08 00:00:00.000000000 Z
+date: 2022-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -278,6 +278,7 @@ files:
 - lib/logstash/plugin_mixins/jdbc/common.rb
 - lib/logstash/plugin_mixins/jdbc/jdbc.rb
 - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
+- lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb
 - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
 - lib/logstash/plugin_mixins/jdbc_streaming.rb
 - lib/logstash/plugin_mixins/jdbc_streaming/cache_payload.rb
@@ -305,6 +306,7 @@ files:
 - spec/helpers/derbyrun.jar
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
+- spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
 - vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
 - vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
@@ -357,4 +359,5 @@ test_files:
 - spec/helpers/derbyrun.jar
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
+- spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb