logstash-integration-jdbc 5.2.4 → 5.4.11

data/spec/inputs/jdbc_spec.rb

@@ -2,13 +2,13 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/devutils/rspec/shared_examples"
 require "logstash/inputs/jdbc"
-require "jdbc/derby"
 require "sequel"
 require "sequel/adapters/jdbc"
 require "timecop"
 require "stud/temporary"
 require "time"
 require "date"
+require "pathname"
 
 # We do not need to set TZ env var anymore because we can have 'Sequel.application_timezone' set to utc by default now.
 
@@ -32,7 +32,6 @@ describe LogStash::Inputs::Jdbc do
   before :each do
     if !RSpec.current_example.metadata[:no_connection]
       # before body
-      Jdbc::Derby.load_driver
       db.create_table :test_table do
         DateTime     :created_at
         BigDecimal   :big_num
@@ -51,6 +50,9 @@ describe LogStash::Inputs::Jdbc do
       db.drop_table(:types_table)
       db.drop_table(:test1_table)
     end
+
+    last_run_default_path = LogStash::SETTINGS.get_value("path.data")
+    FileUtils.rm_f("#{last_run_default_path}/plugins/inputs/jdbc/logstash_jdbc_last_run")
   end
 
   context "when registering and tearing down" do
@@ -247,18 +249,6 @@ describe LogStash::Inputs::Jdbc do
       Timecop.return
     end
 
-    it "cleans up scheduler resources on close" do
-      runner = Thread.new do
-        plugin.run(queue)
-      end
-      sleep 1
-      plugin.do_close
-
-      scheduler = plugin.instance_variable_get(:@scheduler)
-      expect(scheduler).to_not be_nil
-      expect(scheduler.down?).to be_truthy
-    end
-
   end
 
   context "when scheduling and previous runs are to be preserved" do
@@ -285,7 +275,7 @@ describe LogStash::Inputs::Jdbc do
       sleep 1
       for i in 0..1
         sleep 1
-        updated_last_run = YAML.load(File.read(settings["last_run_metadata_path"]))
+        updated_last_run = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))
         expect(updated_last_run).to be > last_run_time
         last_run_time = updated_last_run
       end
@@ -555,7 +545,7 @@ describe LogStash::Inputs::Jdbc do
         expect(actual).to eq(expected)
         plugin.stop
         raw_last_run_value = File.read(settings["last_run_metadata_path"])
-        last_run_value = YAML.load(raw_last_run_value)
+        last_run_value = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(raw_last_run_value)
         expect(last_run_value).to be_a(DateTime)
         expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 02:00:00.722000000 +00:00")
 
@@ -570,7 +560,7 @@ describe LogStash::Inputs::Jdbc do
         plugin.stop
         expect(event.get("num")).to eq(12)
         expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:00:00.811Z"))
-        last_run_value = YAML.load(File.read(settings["last_run_metadata_path"]))
+        last_run_value = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))
         expect(last_run_value).to be_a(DateTime)
         # verify that sub-seconds are recorded to the file
         expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 03:00:00.811000000 +00:00")
@@ -611,6 +601,33 @@ describe LogStash::Inputs::Jdbc do
       # With no timezone set, no change should occur
       expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T12:00:00Z"))
     end
+
+    %w(
+        Etc/UTC
+        America/Los_Angeles
+        Europe/Berlin
+        Asia/Tokyo
+      ).each do |local_timezone|
+      context "when host machine has timezone `#{local_timezone}`" do
+        around(:each) do |example|
+          begin
+            previous_tz = ENV['TZ']
+            ENV['TZ'] = local_timezone
+            example.call
+          ensure
+            ENV['TZ'] = previous_tz
+          end
+        end
+
+        let(:tz) { TZInfo::Timezone.get(local_timezone) }
+
+        it "converts the time using the machine's local timezone" do
+          plugin.run(queue)
+          event = queue.pop
+          expect(event.get("custom_time").time).to eq(Time.new(2015,1,1,12,0,0,tz))
+        end
+      end
+    end
   end
 
   context "when iteratively running plugin#run" do
@@ -704,7 +721,7 @@ describe LogStash::Inputs::Jdbc do
         "last_run_metadata_path" => Stud::Temporary.pathname }
     end
 
-    let(:nums) { [BigDecimal.new(10), BigDecimal.new(20), BigDecimal.new(30), BigDecimal.new(40), BigDecimal.new(50)] }
+    let(:nums) { [BigDecimal(10), BigDecimal(20), BigDecimal(30), BigDecimal(40), BigDecimal(50)] }
 
     before do
       plugin.register
@@ -1114,6 +1131,98 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "when state is persisted" do
+    context "to file" do
+      let(:settings) do
+        {
+          "statement" => "SELECT * FROM test_table",
+          "record_last_run" => true
+        }
+      end
+
+      before do
+        plugin.register
+      end
+
+      after do
+        plugin.stop
+      end
+
+      context "with default last_run_metadata_path" do
+        it "should save state in data.data subpath" do
+          path = LogStash::SETTINGS.get_value("path.data")
+          expect(plugin.last_run_metadata_file_path).to start_with(path)
+        end
+      end
+
+      context "with customized last_run_metadata_path" do
+        let(:settings) { super().merge({ "last_run_metadata_path" => Stud::Temporary.pathname })}
+
+        it "should save state in data.data subpath" do
+          expect(plugin.last_run_metadata_file_path).to start_with(settings["last_run_metadata_path"])
+        end
+      end
+    end
+
+    context "with customized last_run_metadata_path point to directory" do
+      let(:settings) do
+        path = Stud::Temporary.pathname
+        Pathname.new(path).tap {|path| path.mkpath}
+        super().merge({ "last_run_metadata_path" => path})
+      end
+
+      it "raise configuration error" do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
+  context "update the previous default last_run_metadata_path" do
+    let(:settings) do
+      {
+        "statement" => "SELECT * FROM test_table",
+        "record_last_run" => true
+      }
+    end
+
+    let(:fake_home) do
+       path = Stud::Temporary.pathname
+       Pathname.new(path).tap {|path| path.mkpath}
+       path
+    end
+
+    context "when a file exists" do
+      before do
+        # in a faked HOME folder save a valid previous last_run metadata file
+        allow(ENV).to receive(:[]).with(anything).and_call_original
+        allow(ENV).to receive(:[]).with('HOME').and_return(fake_home)
+
+        File.open("#{fake_home}/.logstash_jdbc_last_run", 'w') do |file|
+          file.write("--- !ruby/object:DateTime '2022-03-08 08:10:00.486889000 Z'")
+        end
+      end
+      let(:old_path) { "#{fake_home}/.logstash_jdbc_last_run" }
+      let(:path_data) { LogStash::SETTINGS.get_value("path.data") }
+      let(:new_path) { "#{path_data}/plugins/inputs/jdbc/logstash_jdbc_last_run" }
+
+      it "should be moved" do
+        plugin.register
+        expect(::File).to_not exist(old_path)
+        expect(::File).to exist(new_path)
+      end
+      context "if the delete fails" do
+        before(:each) do
+          allow(File).to receive(:delete).and_raise ArgumentError
+        end
+        it "should be still be moved" do
+          plugin.register
+          expect(::File).to exist(old_path) # old still exists
+          expect(::File).to exist(new_path)
+        end
+      end
+    end
+  end
+
   context "when setting fetch size" do
 
     let(:settings) do
@@ -1280,6 +1389,34 @@ describe LogStash::Inputs::Jdbc do
       expect { plugin.register }.to_not raise_error
       plugin.stop
     end
+
+    it "does retry when query execution fails" do
+      mixin_settings['statement_retry_attempts'] = 2
+      mixin_settings['statement_retry_attempts_wait_time'] = 0.5
+      queue = Queue.new
+      plugin.register
+
+      handler = plugin.instance_variable_get(:@statement_handler)
+      allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_raise(Sequel::PoolTimeout)
+      expect(plugin.logger).to receive(:error).with("Unable to execute statement. Trying again.")
+      expect(plugin.logger).to receive(:error).with("Unable to execute statement. Tried 2 times.")
+
+      plugin.run(queue)
+      plugin.stop
+    end
+
+    it "does not retry when query execution succeeds" do
+      mixin_settings['connection_retry_attempts'] = 2
+      queue = Queue.new
+      plugin.register
+
+      handler = plugin.instance_variable_get(:@statement_handler)
+      allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_call_original
+      expect(plugin.logger).not_to receive(:error)
+
+      plugin.run(queue)
+      plugin.stop
+    end
   end
 
   context "when encoding of some columns need to be changed" do
@@ -1432,57 +1569,64 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
-  context "when debug logging and a count query raises a count related error" do
+  context "when retrieving records with ambiguous timestamps" do
+
     let(:settings) do
-      { "statement" => "SELECT * from types_table" }
-    end
-    let(:logger) { double("logger", :debug? => true) }
-    let(:statement_logger) { LogStash::PluginMixins::Jdbc::CheckedCountLogger.new(logger) }
-    let(:value_tracker) { double("value tracker", :set_value => nil, :write => nil) }
-    let(:msg) { 'Java::JavaSql::SQLSyntaxErrorException: Dynamic SQL Error; SQL error code = -104; Token unknown - line 1, column 105; LIMIT [SQLState:42000, ISC error code:335544634]' }
-    let(:error_args) do
-      {"exception" => msg}
+      {
+        "statement" => "SELECT * from types_table",
+        "jdbc_default_timezone" => jdbc_default_timezone
+      }
     end
 
-    before do
-      db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '1999-12-31 23:59:59', 95.67)"
+    before(:each) do
+      db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '2021-11-07 01:23:45', 95.67)"
       plugin.register
-      plugin.set_statement_logger(statement_logger)
-      plugin.set_value_tracker(value_tracker)
-      allow(value_tracker).to receive(:value).and_return("bar")
-      allow(statement_logger).to receive(:execute_count).once.and_raise(StandardError.new(msg))
     end
 
-    after do
-      plugin.stop
-    end
+    context "when initialized with a preference for DST being enabled" do
+      let(:jdbc_default_timezone) { 'America/Chicago[dst_enabled_on_overlap:true]' }
 
-    context "if the count query raises an error" do
-      it "should log a debug line without a count key as its unknown whether a count works at this stage" do
-        expect(logger).to receive(:warn).once.with("Attempting a count query raised an error, the generated count statement is most likely incorrect but check networking, authentication or your statement syntax", error_args)
-        expect(logger).to receive(:warn).once.with("Ongoing count statement generation is being prevented")
-        expect(logger).to receive(:debug).once.with("Executing JDBC query", :statement => settings["statement"], :parameters => {:sql_last_value=>"bar"})
+      it 'treats the timestamp column as if DST was enabled' do
         plugin.run(queue)
-        queue.pop
+        event = queue.pop
+        expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("2021-11-07T06:23:45Z")
       end
+    end
+    context "when initialized with a preference for DST being disabled" do
+      let(:jdbc_default_timezone) { 'America/Chicago[dst_enabled_on_overlap:false]' }
 
-      it "should create an event normally" do
-        allow(logger).to receive(:warn)
-        allow(logger).to receive(:debug)
+      it 'treats the timestamp column as if DST was disabled' do
         plugin.run(queue)
         event = queue.pop
-        expect(event.get("num")).to eq(1)
-        expect(event.get("string")).to eq("A test")
-        expect(event.get("started_at")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T00:00:00.000Z")
-        expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T23:59:59.000Z")
-        expect(event.get("ranking").to_f).to eq(95.67)
+        expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("2021-11-07T07:23:45Z")
+      end
+    end
+    context "when initialized without a preference for DST being enabled or disabled" do
+      before(:each) { allow(plugin.logger).to receive(:warn) }
+      let(:jdbc_default_timezone) { 'America/Chicago' }
+
+      it 'the error results in helpful log warning' do
+        plugin.run(queue)
+        expect(plugin.logger).to have_received(:warn).with(a_string_including("Exception when executing JDBC query"), a_hash_including(:message => a_string_including("2021-11-07 01:23:45 is an ambiguous local time")))
       end
     end
   end
 
+  def load_derby_version
+    version = {}
+    derby_version =  File.join(Dir.pwd, 'derby_version.txt')
+    File.readlines(derby_version, chomp: true).each do |line|
+      key = line.split('=')[0]
+      value = line.split('=')[1]
+      version[key] = value
+    end
+    version
+  end
+
   context "when an unreadable jdbc_driver_path entry is present" do
     let(:driver_jar_path) do
-      jar_file = $CLASSPATH.find { |name| name.index(Jdbc::Derby.driver_jar) }
+      derby_version = load_derby_version()['DERBY_VERSION']
+      jar_file = $CLASSPATH.find { |name| name.index("derby-#{derby_version}.jar") }
       raise "derby jar not found on class-path" unless jar_file
       jar_file.sub('file:', '')
     end
@@ -1498,16 +1642,15 @@ describe LogStash::Inputs::Jdbc do
       { "statement" => "SELECT * from types_table", "jdbc_driver_library" => invalid_driver_jar_path }
     end
 
-    before do
-      plugin.register
-    end
-
     after do
       plugin.stop
     end
 
     it "raise a loading error" do
-      expect { plugin.run(queue) }.
+      expect(File.exists?(invalid_driver_jar_path)).to be true
+      expect(FileTest.readable?(invalid_driver_jar_path)).to be false
+
+      expect { plugin.register }.
           to raise_error(LogStash::PluginLoadingError, /unable to load .*? from :jdbc_driver_library, file not readable/)
     end
   end
@@ -1619,7 +1762,7 @@ describe LogStash::Inputs::Jdbc do
           plugin.run(queue)
 
           expect(queue.size).to eq(expected_queue_size)
-          expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
+          expect(LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
         end
       end
 
@@ -1644,7 +1787,7 @@ describe LogStash::Inputs::Jdbc do
           plugin.run(queue)
 
           expect(queue.size).to eq(expected_queue_size)
-          expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
+          expect(LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
         end
       end
     end
@@ -1681,7 +1824,7 @@ describe LogStash::Inputs::Jdbc do
       let(:jdbc_driver_class) { "org.apache.NonExistentDriver" }
       it "raises a loading error" do
         expect { plugin.send(:load_driver) }.to raise_error LogStash::PluginLoadingError,
-                                                            /java.lang.ClassNotFoundException: org.apache.NonExistentDriver/
+                                                            /ClassNotFoundException: org.apache.NonExistentDriver/
       end
     end
   end

data/spec/plugin_mixins/jdbc/timezone_proxy_spec.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin_mixins/jdbc/timezone_proxy"
+
+describe LogStash::PluginMixins::Jdbc::TimezoneProxy do
+  subject(:timezone) { described_class.load(timezone_spec) }
+
+  context 'when handling a daylight-savings ambiguous time' do
+    context 'without extensions' do
+      let(:timezone_spec) { 'America/Los_Angeles[]' }
+      it 'raises an AmbiguousTime error' do
+        expect { timezone.local_time(2021,11,7,1,17) }.to raise_error(::TZInfo::AmbiguousTime)
+      end
+    end
+    context 'with extension `dst_enabled_on_overlap:true`' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:true]' }
+      it 'resolves as if DST were enabled' do
+        timestamp = timezone.local_time(2021,11,7,1,17)
+        aggregate_failures do
+          expect(timestamp.dst?).to be true
+          expect(timestamp.zone).to eq('PDT') # Pacific Daylight Time
+          expect(timestamp.getutc).to eq(Time.utc(2021,11,7,8,17))
+          expect(timestamp.utc_offset).to eq( -7 * 3600 )
+        end
+      end
+      end
+    context 'with extension `dst_enabled_on_overlap:false`' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:false]' }
+      it 'resolves as if DST were disabled' do
+        timestamp = timezone.local_time(2021,11,7,1,17)
+        aggregate_failures do
+          expect(timestamp.dst?).to be false
+          expect(timestamp.zone).to eq('PST') # Pacific Standard Time
+          expect(timestamp.getutc).to eq(Time.utc(2021,11,7,9,17))
+          expect(timestamp.utc_offset).to eq( -8 * 3600 )
+        end
+      end
+    end
+  end
+
+  context '#load' do
+    context 'when spec is a normal timezone instance' do
+      let(:timezone_spec) { ::TZInfo::Timezone.get('America/Los_Angeles') }
+      it 'returns that instance' do
+        expect(timezone).to be(timezone_spec)
+      end
+    end
+    context 'when spec is a valid unextended timezone spec' do
+      let(:timezone_spec) { 'America/Los_Angeles' }
+      it 'returns the canonical timezone' do
+        expect(timezone).to eq(::TZInfo::Timezone.get('America/Los_Angeles'))
+      end
+    end
+    context 'when spec is an invalid timezone spec' do
+      let(:timezone_spec) { 'NotAValidTimezoneIdentifier' }
+
+      it 'propagates the TZInfo exception' do
+        expect { timezone }.to raise_exception(::TZInfo::InvalidTimezoneIdentifier)
+      end
+    end
+    context 'with invalid extension' do
+      let(:timezone_spec) { 'America/Los_Angeles[dst_enabled_on_overlap:false;nope:wrong]' }
+      it 'raises an exception with a helpful message' do
+        expect { timezone }.to raise_exception(ArgumentError, a_string_including("Invalid timezone extension `nope:wrong`"))
+      end
+    end
+  end
+end

data/spec/plugin_mixins/jdbc/value_tracking_spec.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+require "logstash/plugin_mixins/jdbc/value_tracking"
+require "tempfile"
+
+module LogStash module PluginMixins module Jdbc
+  describe ValueTracking do
+    context "#load_yaml" do
+
+      context "with date string" do
+        let(:yaml_date_source) { "--- !ruby/object:DateTime '2023-06-15 09:59:30.558000000 +02:00'\n" }
+
+        it "should be loaded" do
+          parsed_date = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_date_source)
+          expect(parsed_date.class).to eq DateTime
+          expect(parsed_date.year).to eq 2023
+          expect(parsed_date.month).to eq 6
+          expect(parsed_date.day).to eq 15
+        end
+      end
+
+      context "with time string" do
+        let(:yaml_time_source) { "--- 2023-06-15 15:28:15.227874000 +02:00\n" }
+
+        it "should be loaded" do
+          parsed_time = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_time_source)
+          expect(parsed_time.class).to eq Time
+          expect(parsed_time.year).to eq 2023
+          expect(parsed_time.month).to eq 6
+          expect(parsed_time.day).to eq 15
+          expect(parsed_time.hour).to eq 15
+          expect(parsed_time.min).to eq 28
+          expect(parsed_time.sec).to eq 15
+        end
+      end
+
+      context "with date string" do
+        let(:yaml_bigdecimal_source) { "--- !ruby/object:BigDecimal '0:0.1e1'\n" }
+
+        it "should be loaded" do
+          parsed_bigdecimal = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_bigdecimal_source)
+          expect(parsed_bigdecimal.class).to eq BigDecimal
+          expect(parsed_bigdecimal.to_i).to eq 1
+        end
+      end
+    end
+
+    context "#build_last_value_tracker" do
+
+      let(:plugin) { double("fake plugin") }
+      let(:temp_file) { Tempfile.new('last_run_tracker') }
+
+      before(:each) do
+        allow(plugin).to receive(:record_last_run).and_return(true)
+        allow(plugin).to receive(:clean_run).and_return(false)
+        allow(plugin).to receive(:last_run_metadata_file_path).and_return(temp_file.path)
+      end
+
+      context "create numerical tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(true)
+          allow(plugin).to receive(:tracking_column_type).and_return("numeric")
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value(1)
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v).to eq 1
+        end
+      end
+
+      context "create date time tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(false)
+          allow(plugin).to receive(:jdbc_default_timezone).and_return(:something_not_nil)
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value("2023-06-15T15:28:15+02:00")
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v.class).to eq DateTime
+          expect(v.year).to eq 2023
+        end
+      end
+
+      context "create time tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(false)
+          allow(plugin).to receive(:jdbc_default_timezone).and_return(nil)
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value("2023-06-15T15:28:15+02:00")
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v.class).to eq Time
+          expect(v.min).to eq 28
+        end
+      end
+
+    end
+  end
+end end end