logstash-integration-jdbc 5.3.0 → 5.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9937a8c1c5ab26acb4cd887f4bdb9dfff51e9348c7187ef2d44ff6055d137ee
-  data.tar.gz: 0042c3e29b5b08abe8baa4f84e4c240960bbc615778df9c77415f8db7985c831
+  metadata.gz: 28cba197157488c1839fc9a948c3f27916b26af07f28c592f537434bd84722a6
+  data.tar.gz: fe2263ea7ed36abdeb9fd3d9597958724be77a8653950e9d554a212c1e3dd106
 SHA512:
-  metadata.gz: dbc38871c106b045a500645cf6c4517b30ce15e52bfc68520845e22276efdb4a21df3e4a7709ae682cd6b55ff76e063a615b9fe397795272535aa7c5298a9a92
-  data.tar.gz: 70b8e1dc84ba60d478affb22c93400cb153d4d0adc4904667970220ac03cb9d22b94e28f1b1a5504a30b8592c543d3234ee441d0ed57a31aefef70dfb6c2b9a3
+  metadata.gz: 76543ead6834631efaca25d154abe7ee7594943dff0c27d90e7588ed3aa651de427fc3372cffac348f6f810f11cfe13864d8f26af7d09fd34470491a6a4c66b3
+  data.tar.gz: de71ae5f8c54dfa08d67e1e848e59fc1779477c41246cee120debf633301829e931b7103546de0bef482002eb7b5296b748d6ad6caf0bb5457abe7205db79b6e

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 5.4.0
+  - Ambiguous Timestamp Support [#92](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/92)
+    - FIX: when encountering an ambiguous timestamp, the JDBC Input no longer crashes
+    - Added support for disambiguating timestamps in daylight saving time (DST) overlap periods
+
 ## 5.3.0
   - Refactor: start using scheduler mixin [#110](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/110)
 

data/docs/filter-jdbc_streaming.asciidoc

@@ -21,7 +21,7 @@ include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
-This filter executes a SQL query and store the result set in the field
+This filter executes a SQL query and stores the result set in the field
 specified as `target`.
 It will cache the results locally in an LRU cache with expiry.
 

data/docs/input-jdbc.asciidoc

@@ -286,10 +286,14 @@ JDBC connection string
 ===== `jdbc_default_timezone`
 
   * Value type is <<string,string>>
+  ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3`
+  ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]`
   * There is no default value for this setting.
 
-Timezone conversion.
-Logstash (and Elasticsearch) expects that timestamps are expressed in UTC terms.
+[id="plugins-{type}s-{plugin}-jdbc_timezone_conv"]
+====== Timezone conversion
+
+Logstash and Elasticsearch expect timestamps to be expressed in UTC terms.
 If your database has recorded timestamps that are relative to another timezone,
 the database timezone if you will, then set this setting to be the timezone that
 the database is using. However, as SQL does not allow for timezone data in
@@ -299,7 +303,15 @@ in relative UTC time in ISO8601 format.
 
 Using this setting will manually assign a specified timezone offset, instead
 of using the timezone setting of the local machine.  You must use a canonical
-timezone, *America/Denver*, for example.
+timezone, `America/Denver`, for example.
+
+[id="plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"]
+===== Ambiguous timestamps
+
+While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline.
+For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous.
+
+When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`).
 
 [id="plugins-{type}s-{plugin}-plugin_timezone"]
 ===== `plugin_timezone`

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -4,6 +4,7 @@ require "logstash/config/mixin"
 require "time"
 require "date"
 require_relative "value_tracking"
+require_relative "timezone_proxy"
 require_relative "statement_handler"
 
 java_import java.util.concurrent.locks.ReentrantLock
@@ -82,7 +83,8 @@ module LogStash  module PluginMixins module Jdbc
       # Using this setting will manually assign a specified timezone offset, instead
       # of using the timezone setting of the local machine.  You must use a canonical
       # timezone, *America/Denver*, for example.
-      config :jdbc_default_timezone, :validate => :string
+      config :jdbc_default_timezone, :validate => :jdbc_timezone_spec
+      extend TimezoneProxy::JDBCTimezoneSpecValidator
 
       # General/Vendor-specific Sequel configuration options.
       #
@@ -157,7 +159,7 @@ module LogStash  module PluginMixins module Jdbc
       @database.extension(:pagination)
       if @jdbc_default_timezone
         @database.extension(:named_timezones)
-        @database.timezone = @jdbc_default_timezone
+        @database.timezone = TimezoneProxy.load(@jdbc_default_timezone)
       end
       if @jdbc_validate_connection
         @database.extension(:connection_validator)
@@ -218,7 +220,10 @@ module LogStash  module PluginMixins module Jdbc
           yield extract_values_from(row)
         end
         success = true
-      rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError, Java::JavaSql::SQLException => e
+      rescue Sequel::DatabaseConnectionError,
+             Sequel::DatabaseError,
+             Sequel::InvalidValue,
+             Java::JavaSql::SQLException => e
         details = { exception: e.class, message: e.message }
         details[:cause] = e.cause.inspect if e.cause
         details[:backtrace] = e.backtrace if @logger.debug?

data/lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+require 'tzinfo'
+
+module LogStash module PluginMixins module Jdbc
+  ##
+  # This `TimezoneProxy` allows timezone specs to include extensions indicating preference for ambiguous handling.
+  # @see TimezoneProxy::parse
+  module TimezoneProxy
+    ##
+    # @param timezone_spec [String]: a timezone spec, consisting of any valid timezone identifier
+    #                                followed by square-bracketed extensions. Currently-supported
+    #                                extensions are:
+    #                                `dst_enabled_on_overlap:(true|false)`: when encountering an ambiguous time
+    #                                                                       due to daylight-savings transition,
+    #                                                                       assume DST to be either enabled or
+    #                                                                       disabled instead of raising an
+    #                                                                       AmbiguousTime exception
+    # @return [TZInfo::Timezone]
+    def self.load(timezone_spec)
+      # re-load pass-through
+      return timezone_spec if timezone_spec.kind_of?(::TZInfo::Timezone)
+
+      parsed_spec = /\A(?<name>[^\[]+)(\[(?<extensions>[^\]]*)\])?\z/.match(timezone_spec)
+
+      timezone = ::TZInfo::Timezone.get(parsed_spec[:name])
+      return timezone unless parsed_spec[:extensions]
+
+      parsed_spec[:extensions].split(';').each do |extension_spec|
+        timezone = case extension_spec
+                   when 'dst_enabled_on_overlap:true'  then timezone.dup.extend(PeriodForLocalWithDSTPreference::ON)
+                   when 'dst_enabled_on_overlap:false' then timezone.dup.extend(PeriodForLocalWithDSTPreference::OFF)
+                   else fail(ArgumentError, "Invalid timezone extension `#{extension_spec}`")
+                   end
+      end
+
+      timezone
+    end
+
+    module JDBCTimezoneSpecValidator
+      def validate_value(value, validator_name)
+        return super(value, validator_name) unless validator_name == :jdbc_timezone_spec
+
+        [true, TimezoneProxy.load(value)] rescue [false, $!.message]
+      end
+    end
+
+    ##
+    # @api private
+    class PeriodForLocalWithDSTPreference < Module
+      def initialize(default_dst_enabled_on_overlap)
+        define_method(:period_for_local) do |localtime, dst_enabled_on_overlap=nil, &dismabiguation_block|
+          super(localtime, dst_enabled_on_overlap.nil? ? default_dst_enabled_on_overlap : dst_enabled_on_overlap, &dismabiguation_block)
+        end
+      end
+
+      ON = new(true)
+      OFF = new(false)
+    end
+  end
+end; end; end

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -14,7 +14,7 @@ module LogStash module PluginMixins module Jdbc
         # use this irrespective of the jdbc_default_timezone setting
         NumericValueTracker.new(handler)
       else
-        if plugin.jdbc_default_timezone.nil? || plugin.jdbc_default_timezone.empty?
+        if plugin.jdbc_default_timezone.nil?
           # no TZ stuff for Sequel, use Time
           TimeValueTracker.new(handler)
         else

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.3.0'
+  s.version         = '5.4.0'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -696,7 +696,7 @@ describe LogStash::Inputs::Jdbc do
         "last_run_metadata_path" => Stud::Temporary.pathname }
     end
 
-    let(:nums) { [BigDecimal.new(10), BigDecimal.new(20), BigDecimal.new(30), BigDecimal.new(40), BigDecimal.new(50)] }
+    let(:nums) { [BigDecimal(10), BigDecimal(20), BigDecimal(30), BigDecimal(40), BigDecimal(50)] }
 
     before do
       plugin.register
@@ -1504,6 +1504,49 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "when retrieving records with ambiguous timestamps" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from types_table",
+        "jdbc_default_timezone" => jdbc_default_timezone
+      }
+    end
+
+    before(:each) do
+      db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '2021-11-07 01:23:45', 95.67)"
+      plugin.register
+    end
+
+    context "when initialized with a preference for DST being enabled" do
+      let(:jdbc_default_timezone) { 'America/Chicago[dst_enabled_on_overlap:true]' }
+
+      it 'treats the timestamp column as if DST was enabled' do
+        plugin.run(queue)
+        event = queue.pop
+        expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("2021-11-07T06:23:45Z")
+      end
+    end
+    context "when initialized with a preference for DST being disabled" do
+      let(:jdbc_default_timezone) { 'America/Chicago[dst_enabled_on_overlap:false]' }
+
+      it 'treats the timestamp column as if DST was disabled' do
+        plugin.run(queue)
+        event = queue.pop
+        expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("2021-11-07T07:23:45Z")
+      end
+    end
+    context "when initialized without a preference for DST being enabled or disabled" do
+      before(:each) { allow(plugin.logger).to receive(:warn) }
+      let(:jdbc_default_timezone) { 'America/Chicago' }
+
+      it 'the error results in helpful log warning' do
+        plugin.run(queue)
+        expect(plugin.logger).to have_received(:warn).with(a_string_including("Exception when executing JDBC query"), a_hash_including(:message => a_string_including("2021-11-07 01:23:45 is an ambiguous local time")))
+      end
+    end
+  end
+
   context "when an unreadable jdbc_driver_path entry is present" do
     let(:driver_jar_path) do
       jar_file = $CLASSPATH.find { |name| name.index(Jdbc::Derby.driver_jar) }

data/spec/plugin_mixins/jdbc/timezone_proxy_spec.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin_mixins/jdbc/timezone_proxy"
+
+describe LogStash::PluginMixins::Jdbc::TimezoneProxy do
+  subject(:timezone) { described_class.load(timezone_spec) }
+
+  context 'when handling a daylight-savings ambiguous time' do
+    context 'without extensions' do
+      let(:timezone_spec) { 'America/Los_Angeles[]' }
+      it 'raises an AmbiguousTime error' do
+        expect { timezone.local_time(2021,11,7,1,17) }.to raise_error(::TZInfo::AmbiguousTime)
+      end
+    end
+    context 'with extension `dst_enabled_on_overlap:true`' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:true]' }
+      it 'resolves as if DST were enabled' do
+        timestamp = timezone.local_time(2021,11,7,1,17)
+        aggregate_failures do
+          expect(timestamp.dst?).to be true
+          expect(timestamp.zone).to eq('PDT') # Pacific Daylight Time
+          expect(timestamp.getutc).to eq(Time.utc(2021,11,7,8,17))
+          expect(timestamp.utc_offset).to eq( -7 * 3600 )
+        end
+      end
+      end
+    context 'with extension `dst_enabled_on_overlap:false`' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:false]' }
+      it 'resolves as if DST were disabled' do
+        timestamp = timezone.local_time(2021,11,7,1,17)
+        aggregate_failures do
+          expect(timestamp.dst?).to be false
+          expect(timestamp.zone).to eq('PST') # Pacific Standard Time
+          expect(timestamp.getutc).to eq(Time.utc(2021,11,7,9,17))
+          expect(timestamp.utc_offset).to eq( -8 * 3600 )
+        end
+      end
+    end
+  end
+
+  context '#load' do
+    context 'when spec is a normal timezone instance' do
+      let(:timezone_spec) { ::TZInfo::Timezone.get('America/Los_Angeles') }
+      it 'returns that instance' do
+        expect(timezone).to be(timezone_spec)
+      end
+    end
+    context 'when spec is a valid unextended timezone spec' do
+      let(:timezone_spec) { 'America/Los_Angeles' }
+      it 'returns the canonical timezone' do
+        expect(timezone).to eq(::TZInfo::Timezone.get('America/Los_Angeles'))
+      end
+    end
+    context 'when spec is an invalid timezone spec' do
+      let(:timezone_spec) { 'NotAValidTimezoneIdentifier' }
+
+      it 'propagates the TZInfo exception' do
+        expect { timezone }.to raise_exception(::TZInfo::InvalidTimezoneIdentifier)
+      end
+    end
+    context 'with invalid extension' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:false;nope:wrong]' }
+      it 'raises an exception with a helpful message' do
+        expect { timezone }.to raise_exception(ArgumentError, a_string_including("Invalid timezone extension `nope:wrong`"))
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.3.0
+  version: 5.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-08 00:00:00.000000000 Z
+date: 2022-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -278,6 +278,7 @@ files:
 - lib/logstash/plugin_mixins/jdbc/common.rb
 - lib/logstash/plugin_mixins/jdbc/jdbc.rb
 - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
+- lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb
 - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
 - lib/logstash/plugin_mixins/jdbc_streaming.rb
 - lib/logstash/plugin_mixins/jdbc_streaming/cache_payload.rb
@@ -305,6 +306,7 @@ files:
 - spec/helpers/derbyrun.jar
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
+- spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
 - vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
 - vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
@@ -357,4 +359,5 @@ test_files:
 - spec/helpers/derbyrun.jar
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
+- spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb