logstash-integration-jdbc 5.3.0 → 5.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d9937a8c1c5ab26acb4cd887f4bdb9dfff51e9348c7187ef2d44ff6055d137ee
4
- data.tar.gz: 0042c3e29b5b08abe8baa4f84e4c240960bbc615778df9c77415f8db7985c831
3
+ metadata.gz: 28cba197157488c1839fc9a948c3f27916b26af07f28c592f537434bd84722a6
4
+ data.tar.gz: fe2263ea7ed36abdeb9fd3d9597958724be77a8653950e9d554a212c1e3dd106
5
5
  SHA512:
6
- metadata.gz: dbc38871c106b045a500645cf6c4517b30ce15e52bfc68520845e22276efdb4a21df3e4a7709ae682cd6b55ff76e063a615b9fe397795272535aa7c5298a9a92
7
- data.tar.gz: 70b8e1dc84ba60d478affb22c93400cb153d4d0adc4904667970220ac03cb9d22b94e28f1b1a5504a30b8592c543d3234ee441d0ed57a31aefef70dfb6c2b9a3
6
+ metadata.gz: 76543ead6834631efaca25d154abe7ee7594943dff0c27d90e7588ed3aa651de427fc3372cffac348f6f810f11cfe13864d8f26af7d09fd34470491a6a4c66b3
7
+ data.tar.gz: de71ae5f8c54dfa08d67e1e848e59fc1779477c41246cee120debf633301829e931b7103546de0bef482002eb7b5296b748d6ad6caf0bb5457abe7205db79b6e
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 5.4.0
2
+ - Ambiguous Timestamp Support [#92](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/92)
3
+ - FIX: when encountering an ambiguous timestamp, the JDBC Input no longer crashes
4
+ - Added support for disambiguating timestamps in daylight saving time (DST) overlap periods
5
+
1
6
  ## 5.3.0
2
7
  - Refactor: start using scheduler mixin [#110](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/110)
3
8
 
@@ -21,7 +21,7 @@ include::{include_path}/plugin_header-integration.asciidoc[]
21
21
 
22
22
  ==== Description
23
23
 
24
- This filter executes a SQL query and store the result set in the field
24
+ This filter executes a SQL query and stores the result set in the field
25
25
  specified as `target`.
26
26
  It will cache the results locally in an LRU cache with expiry.
27
27
 
@@ -286,10 +286,14 @@ JDBC connection string
286
286
  ===== `jdbc_default_timezone`
287
287
 
288
288
  * Value type is <<string,string>>
289
+ ** Value should be a canonical timezone or offset, such as `Europe/Paris` or `Etc/GMT+3`
290
+ ** Value _may_ include square-bracketed extensions, such as `America/Denver[dst_enabled_on_overlap:true]`
289
291
  * There is no default value for this setting.
290
292
 
291
- Timezone conversion.
292
- Logstash (and Elasticsearch) expects that timestamps are expressed in UTC terms.
293
+ [id="plugins-{type}s-{plugin}-jdbc_timezone_conv"]
294
+ ====== Timezone conversion
295
+
296
+ Logstash and Elasticsearch expect timestamps to be expressed in UTC terms.
293
297
  If your database has recorded timestamps that are relative to another timezone,
294
298
  the database timezone if you will, then set this setting to be the timezone that
295
299
  the database is using. However, as SQL does not allow for timezone data in
@@ -299,7 +303,15 @@ in relative UTC time in ISO8601 format.
299
303
 
300
304
  Using this setting will manually assign a specified timezone offset, instead
301
305
  of using the timezone setting of the local machine. You must use a canonical
302
- timezone, *America/Denver*, for example.
306
+ timezone, `America/Denver`, for example.
307
+
308
+ [id="plugins-{type}s-{plugin}-jdbc_ambiguous_timestamps"]
309
+ ===== Ambiguous timestamps
310
+
311
+ While it is common to store local times in SQL's timestamp column type, many timezones change their offset during the course of a calendar year and therefore cannot be used with SQL's timestamp type to represent an ordered, continuous timeline.
312
+ For example in the `America/Chicago` zone when daylight saving time (DST) ends in the autumn, the clock rolls from `01:59:59` back to `01:00:00`, making any timestamp in the 2-hour period between `01:00:00CDT` and `02:00:00CST` on that day ambiguous.
313
+
314
+ When encountering an ambiguous timestamp caused by a DST transition, the query will fail unless the timezone specified here includes a square-bracketed instruction for how to handle overlapping periods (such as: `America/Chicago[dst_enabled_on_overlap:true]` or `Australia/Melbourne[dst_enabled_on_overlap:false]`).
303
315
 
304
316
  [id="plugins-{type}s-{plugin}-plugin_timezone"]
305
317
  ===== `plugin_timezone`
@@ -4,6 +4,7 @@ require "logstash/config/mixin"
4
4
  require "time"
5
5
  require "date"
6
6
  require_relative "value_tracking"
7
+ require_relative "timezone_proxy"
7
8
  require_relative "statement_handler"
8
9
 
9
10
  java_import java.util.concurrent.locks.ReentrantLock
@@ -82,7 +83,8 @@ module LogStash module PluginMixins module Jdbc
82
83
  # Using this setting will manually assign a specified timezone offset, instead
83
84
  # of using the timezone setting of the local machine. You must use a canonical
84
85
  # timezone, *America/Denver*, for example.
85
- config :jdbc_default_timezone, :validate => :string
86
+ config :jdbc_default_timezone, :validate => :jdbc_timezone_spec
87
+ extend TimezoneProxy::JDBCTimezoneSpecValidator
86
88
 
87
89
  # General/Vendor-specific Sequel configuration options.
88
90
  #
@@ -157,7 +159,7 @@ module LogStash module PluginMixins module Jdbc
157
159
  @database.extension(:pagination)
158
160
  if @jdbc_default_timezone
159
161
  @database.extension(:named_timezones)
160
- @database.timezone = @jdbc_default_timezone
162
+ @database.timezone = TimezoneProxy.load(@jdbc_default_timezone)
161
163
  end
162
164
  if @jdbc_validate_connection
163
165
  @database.extension(:connection_validator)
@@ -218,7 +220,10 @@ module LogStash module PluginMixins module Jdbc
218
220
  yield extract_values_from(row)
219
221
  end
220
222
  success = true
221
- rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError, Java::JavaSql::SQLException => e
223
+ rescue Sequel::DatabaseConnectionError,
224
+ Sequel::DatabaseError,
225
+ Sequel::InvalidValue,
226
+ Java::JavaSql::SQLException => e
222
227
  details = { exception: e.class, message: e.message }
223
228
  details[:cause] = e.cause.inspect if e.cause
224
229
  details[:backtrace] = e.backtrace if @logger.debug?
@@ -0,0 +1,61 @@
1
+ # encoding: utf-8
2
+
3
+ require 'tzinfo'
4
+
5
+ module LogStash module PluginMixins module Jdbc
6
+ ##
7
+ # This `TimezoneProxy` allows timezone specs to include extensions indicating preference for ambiguous handling.
8
+ # @see TimezoneProxy::parse
9
+ module TimezoneProxy
10
+ ##
11
+ # @param timezone_spec [String]: a timezone spec, consisting of any valid timezone identifier
12
+ # followed by square-bracketed extensions. Currently-supported
13
+ # extensions are:
14
+ # `dst_enabled_on_overlap:(true|false)`: when encountering an ambiguous time
15
+ # due to daylight-savings transition,
16
+ # assume DST to be either enabled or
17
+ # disabled instead of raising an
18
+ # AmbiguousTime exception
19
+ # @return [TZInfo::Timezone]
20
+ def self.load(timezone_spec)
21
+ # re-load pass-through
22
+ return timezone_spec if timezone_spec.kind_of?(::TZInfo::Timezone)
23
+
24
+ parsed_spec = /\A(?<name>[^\[]+)(\[(?<extensions>[^\]]*)\])?\z/.match(timezone_spec)
25
+
26
+ timezone = ::TZInfo::Timezone.get(parsed_spec[:name])
27
+ return timezone unless parsed_spec[:extensions]
28
+
29
+ parsed_spec[:extensions].split(';').each do |extension_spec|
30
+ timezone = case extension_spec
31
+ when 'dst_enabled_on_overlap:true' then timezone.dup.extend(PeriodForLocalWithDSTPreference::ON)
32
+ when 'dst_enabled_on_overlap:false' then timezone.dup.extend(PeriodForLocalWithDSTPreference::OFF)
33
+ else fail(ArgumentError, "Invalid timezone extension `#{extension_spec}`")
34
+ end
35
+ end
36
+
37
+ timezone
38
+ end
39
+
40
+ module JDBCTimezoneSpecValidator
41
+ def validate_value(value, validator_name)
42
+ return super(value, validator_name) unless validator_name == :jdbc_timezone_spec
43
+
44
+ [true, TimezoneProxy.load(value)] rescue [false, $!.message]
45
+ end
46
+ end
47
+
48
+ ##
49
+ # @api private
50
+ class PeriodForLocalWithDSTPreference < Module
51
+ def initialize(default_dst_enabled_on_overlap)
52
+ define_method(:period_for_local) do |localtime, dst_enabled_on_overlap=nil, &dismabiguation_block|
53
+ super(localtime, dst_enabled_on_overlap.nil? ? default_dst_enabled_on_overlap : dst_enabled_on_overlap, &dismabiguation_block)
54
+ end
55
+ end
56
+
57
+ ON = new(true)
58
+ OFF = new(false)
59
+ end
60
+ end
61
+ end; end; end
@@ -14,7 +14,7 @@ module LogStash module PluginMixins module Jdbc
14
14
  # use this irrespective of the jdbc_default_timezone setting
15
15
  NumericValueTracker.new(handler)
16
16
  else
17
- if plugin.jdbc_default_timezone.nil? || plugin.jdbc_default_timezone.empty?
17
+ if plugin.jdbc_default_timezone.nil?
18
18
  # no TZ stuff for Sequel, use Time
19
19
  TimeValueTracker.new(handler)
20
20
  else
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-integration-jdbc'
3
- s.version = '5.3.0'
3
+ s.version = '5.4.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Integration with JDBC - input and filter plugins"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -696,7 +696,7 @@ describe LogStash::Inputs::Jdbc do
696
696
  "last_run_metadata_path" => Stud::Temporary.pathname }
697
697
  end
698
698
 
699
- let(:nums) { [BigDecimal.new(10), BigDecimal.new(20), BigDecimal.new(30), BigDecimal.new(40), BigDecimal.new(50)] }
699
+ let(:nums) { [BigDecimal(10), BigDecimal(20), BigDecimal(30), BigDecimal(40), BigDecimal(50)] }
700
700
 
701
701
  before do
702
702
  plugin.register
@@ -1504,6 +1504,49 @@ describe LogStash::Inputs::Jdbc do
1504
1504
  end
1505
1505
  end
1506
1506
 
1507
+ context "when retrieving records with ambiguous timestamps" do
1508
+
1509
+ let(:settings) do
1510
+ {
1511
+ "statement" => "SELECT * from types_table",
1512
+ "jdbc_default_timezone" => jdbc_default_timezone
1513
+ }
1514
+ end
1515
+
1516
+ before(:each) do
1517
+ db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '2021-11-07 01:23:45', 95.67)"
1518
+ plugin.register
1519
+ end
1520
+
1521
+ context "when initialized with a preference for DST being enabled" do
1522
+ let(:jdbc_default_timezone) { 'America/Chicago[dst_enabled_on_overlap:true]' }
1523
+
1524
+ it 'treats the timestamp column as if DST was enabled' do
1525
+ plugin.run(queue)
1526
+ event = queue.pop
1527
+ expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("2021-11-07T06:23:45Z")
1528
+ end
1529
+ end
1530
+ context "when initialized with a preference for DST being disabled" do
1531
+ let(:jdbc_default_timezone) { 'America/Chicago[dst_enabled_on_overlap:false]' }
1532
+
1533
+ it 'treats the timestamp column as if DST was disabled' do
1534
+ plugin.run(queue)
1535
+ event = queue.pop
1536
+ expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("2021-11-07T07:23:45Z")
1537
+ end
1538
+ end
1539
+ context "when initialized without a preference for DST being enabled or disabled" do
1540
+ before(:each) { allow(plugin.logger).to receive(:warn) }
1541
+ let(:jdbc_default_timezone) { 'America/Chicago' }
1542
+
1543
+ it 'the error results in helpful log warning' do
1544
+ plugin.run(queue)
1545
+ expect(plugin.logger).to have_received(:warn).with(a_string_including("Exception when executing JDBC query"), a_hash_including(:message => a_string_including("2021-11-07 01:23:45 is an ambiguous local time")))
1546
+ end
1547
+ end
1548
+ end
1549
+
1507
1550
  context "when an unreadable jdbc_driver_path entry is present" do
1508
1551
  let(:driver_jar_path) do
1509
1552
  jar_file = $CLASSPATH.find { |name| name.index(Jdbc::Derby.driver_jar) }
@@ -0,0 +1,68 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/plugin_mixins/jdbc/timezone_proxy"
4
+
5
+ describe LogStash::PluginMixins::Jdbc::TimezoneProxy do
6
+ subject(:timezone) { described_class.load(timezone_spec) }
7
+
8
+ context 'when handling a daylight-savings ambiguous time' do
9
+ context 'without extensions' do
10
+ let(:timezone_spec) { 'America/Los_Angeles[]' }
11
+ it 'raises an AmbiguousTime error' do
12
+ expect { timezone.local_time(2021,11,7,1,17) }.to raise_error(::TZInfo::AmbiguousTime)
13
+ end
14
+ end
15
+ context 'with extension `dst_enabled_on_overlap:true`' do
16
+ let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:true]' }
17
+ it 'resolves as if DST were enabled' do
18
+ timestamp = timezone.local_time(2021,11,7,1,17)
19
+ aggregate_failures do
20
+ expect(timestamp.dst?).to be true
21
+ expect(timestamp.zone).to eq('PDT') # Pacific Daylight Time
22
+ expect(timestamp.getutc).to eq(Time.utc(2021,11,7,8,17))
23
+ expect(timestamp.utc_offset).to eq( -7 * 3600 )
24
+ end
25
+ end
26
+ end
27
+ context 'with extension `dst_enabled_on_overlap:false`' do
28
+ let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:false]' }
29
+ it 'resolves as if DST were disabled' do
30
+ timestamp = timezone.local_time(2021,11,7,1,17)
31
+ aggregate_failures do
32
+ expect(timestamp.dst?).to be false
33
+ expect(timestamp.zone).to eq('PST') # Pacific Standard Time
34
+ expect(timestamp.getutc).to eq(Time.utc(2021,11,7,9,17))
35
+ expect(timestamp.utc_offset).to eq( -8 * 3600 )
36
+ end
37
+ end
38
+ end
39
+ end
40
+
41
+ context '#load' do
42
+ context 'when spec is a normal timezone instance' do
43
+ let(:timezone_spec) { ::TZInfo::Timezone.get('America/Los_Angeles') }
44
+ it 'returns that instance' do
45
+ expect(timezone).to be(timezone_spec)
46
+ end
47
+ end
48
+ context 'when spec is a valid unextended timezone spec' do
49
+ let(:timezone_spec) { 'America/Los_Angeles' }
50
+ it 'returns the canonical timezone' do
51
+ expect(timezone).to eq(::TZInfo::Timezone.get('America/Los_Angeles'))
52
+ end
53
+ end
54
+ context 'when spec is an invalid timezone spec' do
55
+ let(:timezone_spec) { 'NotAValidTimezoneIdentifier' }
56
+
57
+ it 'propagates the TZInfo exception' do
58
+ expect { timezone }.to raise_exception(::TZInfo::InvalidTimezoneIdentifier)
59
+ end
60
+ end
61
+ context 'with invalid extension' do
62
+ let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:false;nope:wrong]' }
63
+ it 'raises an exception with a helpful message' do
64
+ expect { timezone }.to raise_exception(ArgumentError, a_string_including("Invalid timezone extension `nope:wrong`"))
65
+ end
66
+ end
67
+ end
68
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-integration-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.3.0
4
+ version: 5.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-06-08 00:00:00.000000000 Z
11
+ date: 2022-10-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -278,6 +278,7 @@ files:
278
278
  - lib/logstash/plugin_mixins/jdbc/common.rb
279
279
  - lib/logstash/plugin_mixins/jdbc/jdbc.rb
280
280
  - lib/logstash/plugin_mixins/jdbc/statement_handler.rb
281
+ - lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb
281
282
  - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
282
283
  - lib/logstash/plugin_mixins/jdbc_streaming.rb
283
284
  - lib/logstash/plugin_mixins/jdbc_streaming/cache_payload.rb
@@ -305,6 +306,7 @@ files:
305
306
  - spec/helpers/derbyrun.jar
306
307
  - spec/inputs/integration/integ_spec.rb
307
308
  - spec/inputs/jdbc_spec.rb
309
+ - spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
308
310
  - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
309
311
  - vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
310
312
  - vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
@@ -357,4 +359,5 @@ test_files:
357
359
  - spec/helpers/derbyrun.jar
358
360
  - spec/inputs/integration/integ_spec.rb
359
361
  - spec/inputs/jdbc_spec.rb
362
+ - spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
360
363
  - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb