logstash-input-jdbc 4.3.4 → 4.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4182f11b49a8109b280f7c078b547a71b3c7134dc54834b06705f94ba064d216
-  data.tar.gz: 6decb981a1ce99e9aac59da1b0259968db299e348d16c76a83cac07e87a58fa8
+  metadata.gz: f382c4428bc37a74fdfae0aac1a97eca218555266c01cb76a4c6f6de5e7e3ad1
+  data.tar.gz: 70d0027fac72146ee098c78f46c7bac8f3d8feb2139f9e6238eb44da1ba3de8f
 SHA512:
-  metadata.gz: 8d9cb81fe24f6f301166f2f8baeaa90f18e43f4d3d7ef0ff9c09516d9edd57a5bf6e4eaa9d3a112af7616b062b39f781384dbc25ee3ef9a20fbc1594e36612ca
-  data.tar.gz: e2b979fdfc781cda71aaa0aa1232f9c25b21e120f89ec07729e3bb41e68a57cd7d6d8c467a6db79bf66a83d2f9f29209d2ed76877fe1595af0ee6e2190649152
+  metadata.gz: 471770e1b701221c9e0b5897c00313276a28f138a5831c9f1a027ca0655951115542a19edc32a4f4bbe634d9a96f969cc45e9d48d4269f72a1780ad6063b468b
+  data.tar.gz: af300cbd6b078c2d9efcfefc5482480a21e020708fd8e71036c234183dd81e734ab1775b69c9810f0f8e1d13d04dae485979b46857d8d7d773a533b9413b550b

data/CHANGELOG.md

@@ -1,17 +1,20 @@
+## 4.3.5
+  - [#140](https://github.com/logstash-plugins/logstash-input-jdbc/issues/140) Fix long standing bug where setting jdbc_default_timezone loses milliseconds. Force all usage of sql_last_value to be typed according to the settings.
+
 ## 4.3.4
   - [#261](https://github.com/logstash-plugins/logstash-input-jdbc/issues/261) Fix memory leak.
-  
+
 ## 4.3.3
   - [#255](https://github.com/logstash-plugins/logstash-input-jdbc/issues/255) Fix thread and memory leak.
-  
+
 ## 4.3.2
   - [#251](https://github.com/logstash-plugins/logstash-input-jdbc/issues/251) Fix connection and memory leak.
-  
+
 ## 4.3.1
   - Update gemspec summary
 
-## 4.3.0  
-  - [#147](https://github.com/logstash-plugins/logstash-input-jdbc/issues/147) Open and close connection for each query 
+## 4.3.0
+  - [#147](https://github.com/logstash-plugins/logstash-input-jdbc/issues/147) Open and close connection for each query
 
 ## 4.2.4
   - [#220](https://github.com/logstash-plugins/logstash-input-jdbc/issues/220) Log exception when database connection test fails

data/lib/logstash/inputs/jdbc.rb

@@ -2,7 +2,7 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/plugin_mixins/jdbc"
-require "yaml" # persistence
+
 
 # This plugin was created as a way to ingest data from any database
 # with a JDBC interface into Logstash. You can periodically schedule ingestion
@@ -206,21 +206,16 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     require "rufus/scheduler"
     prepare_jdbc_connection
 
-    # Raise an error if @use_column_value is true, but no @tracking_column is set
     if @use_column_value
+      # Raise an error if @use_column_value is true, but no @tracking_column is set
       if @tracking_column.nil?
         raise(LogStash::ConfigurationError, "Must set :tracking_column if :use_column_value is true.")
       end
     end
 
-    @enable_encoding = !@charset.nil? || !@columns_charset.empty?
+    @value_tracker = LogStash::PluginMixins::ValueTracking.build_last_value_tracker(self)
 
-    # load sql_last_value from file if exists
-    if @clean_run && File.exist?(@last_run_metadata_path)
-      File.delete(@last_run_metadata_path)
-    elsif File.exist?(@last_run_metadata_path)
-      @sql_last_value = YAML.load(File.read(@last_run_metadata_path))
-    end
+    @enable_encoding = !@charset.nil? || !@columns_charset.empty?
 
     unless @statement.nil? ^ @statement_filepath.nil?
       raise(LogStash::ConfigurationError, "Must set either :statement or :statement_filepath. Only one may be set at a time.")
@@ -248,13 +243,11 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
       @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
       @scheduler.cron @schedule do
         execute_query(queue)
-        update_state_file
       end
 
       @scheduler.join
     else
       execute_query(queue)
-      update_state_file
     end
   end # def run
 
@@ -267,7 +260,7 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
 
   def execute_query(queue)
     # update default parameters
-    @parameters['sql_last_value'] = @sql_last_value
+    @parameters['sql_last_value'] = @value_tracker.value
     execute_statement(@statement, @parameters) do |row|
       if enable_encoding?
         ## do the necessary conversions to string elements
@@ -277,12 +270,7 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
       decorate(event)
       queue << event
     end
-  end
-
-  def update_state_file
-    if @record_last_run
-      File.write(@last_run_metadata_path, YAML.dump(@sql_last_value))
-    end
+    @value_tracker.write
   end
 
   private

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -3,6 +3,7 @@
 require "logstash/config/mixin"
 require "time"
 require "date"
+require "logstash/plugin_mixins/value_tracking"
 
 java_import java.util.concurrent.locks.ReentrantLock
 
@@ -195,17 +196,7 @@ module LogStash::PluginMixins::Jdbc
   public
   def prepare_jdbc_connection
     @connection_lock = ReentrantLock.new
-    if @use_column_value
-      case @tracking_column_type
-        when "numeric"
-          @sql_last_value = 0
-        when "timestamp"
-          @sql_last_value = Time.at(0).utc
-      end
-    else
-      @sql_last_value = Time.at(0).utc
-    end
-  end # def prepare_jdbc_connection
+  end
 
   public
   def close_jdbc_connection
@@ -229,22 +220,20 @@ module LogStash::PluginMixins::Jdbc
       begin
         parameters = symbolized_params(parameters)
         query = @database[statement, parameters]
-        sql_last_value = @use_column_value ? @sql_last_value : Time.now.utc
+
+        sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
         @tracking_column_warning_sent = false
         @logger.debug? and @logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters, :count => query.count)
 
         perform_query(query) do |row|
           sql_last_value = get_column_value(row) if @use_column_value
-          if @tracking_column_type=="timestamp" and @use_column_value and sql_last_value.is_a?(DateTime)
-            sql_last_value = sql_last_value.to_time # Coerce the timestamp to a `Time`
-          end
           yield extract_values_from(row)
         end
         success = true
       rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError => e
         @logger.warn("Exception when executing JDBC query", :exception => e)
       else
-        @sql_last_value = sql_last_value
+        @value_tracker.set_value(sql_last_value)
       ensure
         close_jdbc_connection
         @connection_lock.unlock

data/lib/logstash/plugin_mixins/value_tracking.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+require "yaml" # persistence
+
+module LogStash module PluginMixins
+  class ValueTracking
+
+    def self.build_last_value_tracker(plugin)
+      if plugin.use_column_value && plugin.tracking_column_type == "numeric"
+        # use this irrespective of the jdbc_default_timezone setting
+        klass = NumericValueTracker
+      else
+        if plugin.jdbc_default_timezone.nil? || plugin.jdbc_default_timezone.empty?
+          # no TZ stuff for Sequel, use Time
+          klass = TimeValueTracker
+        else
+          # Sequel does timezone handling on DateTime only
+          klass = DateTimeValueTracker
+        end
+      end
+
+      handler = NullFileHandler.new(plugin.last_run_metadata_path)
+      if plugin.record_last_run
+        handler = FileHandler.new(plugin.last_run_metadata_path)
+      end
+      if plugin.clean_run
+        handler.clean
+      end
+
+      instance = klass.new(handler)
+    end
+
+    attr_reader :value
+
+    def initialize(handler)
+      @file_handler = handler
+      set_value(get_initial)
+    end
+
+    def get_initial
+      # override in subclass
+    end
+
+    def set_value(value)
+      # override in subclass
+    end
+
+    def write
+      @file_handler.write(@value)
+    end
+  end
+
+
+  class NumericValueTracker < ValueTracking
+    def get_initial
+      @file_handler.read || 0
+    end
+
+    def set_value(value)
+      return unless value.is_a?(Numeric)
+      @value = value
+    end
+  end
+
+  class DateTimeValueTracker < ValueTracking
+    def get_initial
+      @file_handler.read || DateTime.new(1970)
+    end
+
+    def set_value(value)
+      if value.respond_to?(:to_datetime)
+        @value = value.to_datetime
+      end
+    end
+  end
+
+  class TimeValueTracker < ValueTracking
+    def get_initial
+      @file_handler.read || Time.at(0).utc
+    end
+
+    def set_value(value)
+      if value.respond_to?(:to_time)
+        @value = value.to_time
+      end
+    end
+  end
+
+  class FileHandler
+    def initialize(path)
+      @path = path
+      @exists = ::File.exist?(@path)
+    end
+
+    def clean
+      return unless @exists
+      ::File.delete(@path)
+      @exists = false
+    end
+
+    def read
+      return unless @exists
+      YAML.load(::File.read(@path))
+    end
+
+    def write(value)
+      ::File.write(@path, YAML.dump(value))
+      @exists = true
+    end
+  end
+
+  class NullFileHandler
+    def initialize(path)
+    end
+
+    def clean
+    end
+
+    def read
+    end
+
+    def write(value)
+    end
+  end
+end end

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '4.3.4'
+  s.version         = '4.3.5'
   s.licenses = ['Apache License (2.0)']
   s.summary = "Creates events from JDBC data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -38,6 +38,7 @@ describe LogStash::Inputs::Jdbc do
         DateTime :custom_time
       end
       db << "CREATE TABLE types_table (num INTEGER, string VARCHAR(255), started_at DATE, custom_time TIMESTAMP, ranking DECIMAL(16,6))"
+      db << "CREATE TABLE test1_table (num INTEGER, string VARCHAR(255), custom_time TIMESTAMP, created_at TIMESTAMP)"
     end
   end
 
@@ -45,6 +46,7 @@ describe LogStash::Inputs::Jdbc do
     if !RSpec.current_example.metadata[:no_connection]
       db.drop_table(:test_table)
       db.drop_table(:types_table)
+      db.drop_table(:test1_table)
     end
   end
 
@@ -322,7 +324,7 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
-  context "when fetching time data with jdbc_default_timezone set" do
+  describe "when jdbc_default_timezone is set" do
     let(:mixin_settings) do
       { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
         "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true",
@@ -330,53 +332,136 @@ describe LogStash::Inputs::Jdbc do
       }
     end
 
-    let(:settings) do
-      {
-        "statement" => "SELECT * from test_table WHERE custom_time > :sql_last_value",
-        "use_column_value" => true,
-        "tracking_column" => "custom_time",
-        "last_run_metadata_path" => Stud::Temporary.pathname
-      }
+    let(:hours) { [10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] }
+
+    context "when fetching time data and the tracking column is set and tracking column type defaults to 'numeric'" do
+      let(:settings) do
+        {
+          "statement" => "SELECT * from test_table WHERE num > :sql_last_value",
+          "last_run_metadata_path" => Stud::Temporary.pathname,
+          "tracking_column" => "num",
+          "use_column_value" => true
+        }
+      end
+
+      it "should convert the time to reflect the timezone " do
+        File.write(settings["last_run_metadata_path"], YAML.dump(42))
+
+        db[:test_table].insert(:num => 42, :custom_time => "2015-01-01 10:10:10", :created_at => Time.now.utc)
+        db[:test_table].insert(:num => 43, :custom_time => "2015-01-01 11:11:11", :created_at => Time.now.utc)
+
+        plugin.register
+        plugin.run(queue)
+        plugin.stop
+        expect(queue.size).to eq(1)
+        event = queue.pop
+        expect(event.get("num")).to eq(43)
+        expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T17:11:11.000Z"))
+      end
     end
 
-    let(:hour_range) { 10..20 }
+    context "when fetching time data and the tracking column is NOT set, sql_last_value is time of run" do
 
-    it "should convert the time to reflect the timezone " do
-      last_run_value = Time.iso8601("2000-01-01T00:00:00.000Z")
-      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+      let(:settings) do
+        {
+          "statement" => "SELECT * from test_table WHERE custom_time > :sql_last_value",
+          "last_run_metadata_path" => Stud::Temporary.pathname
+        }
+      end
+
+      it "should convert the time to reflect the timezone " do
+        last_run_value = DateTime.iso8601("2000-01-01T12:00:00.000Z")
+        File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+
+        Timecop.travel(DateTime.iso8601("2015-01-01T15:50:00.000Z")) do
+          # simulate earlier records written
+          hours.each do |i|
+            db[:test_table].insert(:num => i, :custom_time => "2015-01-01 #{i}:00:00", :created_at => Time.now.utc)
+          end
+        end
 
-      hour_range.each do |i|
-        db[:test_table].insert(:num => i, :custom_time => "2015-01-01 #{i}:00:00", :created_at => Time.now.utc)
+        Timecop.travel(DateTime.iso8601("2015-01-02T02:10:00.000Z")) do
+          # simulate the first plugin run after the custom time of the last record
+          plugin.register
+          plugin.run(queue)
+          expected = hours.map{|hour| Time.iso8601("2015-01-01T06:00:00.000Z") + (hour * 3600) }# because Sequel converts the column values to Time instances.
+          actual = queue.size.times.map { queue.pop.get("custom_time").time }
+          expect(actual).to eq(expected)
+          plugin.stop
+        end
+        Timecop.travel(DateTime.iso8601("2015-01-02T02:20:00.000Z")) do
+          # simulate a run 10 minutes later
+          plugin.run(queue)
+          expect(queue.size).to eq(0) # no new records
+          plugin.stop
+          # now add records
+          db[:test_table].insert(:num => 11, :custom_time => "2015-01-01 20:20:20", :created_at => Time.now.utc)
+          db[:test_table].insert(:num => 12, :custom_time => "2015-01-01 21:21:21", :created_at => Time.now.utc)
+        end
+        Timecop.travel(DateTime.iso8601("2015-01-02T03:30:00.000Z")) do
+          # simulate another run later than the custom time of the last record
+          plugin.run(queue)
+          expect(queue.size).to eq(2)
+          plugin.stop
+        end
+        event = queue.pop
+        expect(event.get("num")).to eq(11)
+        expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T02:20:20.000Z"))
+        event = queue.pop
+        expect(event.get("num")).to eq(12)
+        expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:21:21.000Z"))
       end
+    end
 
-      plugin.register
+    context "when fetching time data and the tracking column is set, sql_last_value is sourced from a column, sub-second precision is maintained" do
+      let(:settings) do
+        {
+          "statement" => "SELECT * from test1_table WHERE custom_time > :sql_last_value ORDER BY custom_time",
+          "use_column_value" => true,
+          "tracking_column" => "custom_time",
+          "tracking_column_type" => "timestamp",
+          "last_run_metadata_path" => Stud::Temporary.pathname
+        }
+      end
 
-      plugin.run(queue)
-      expected = ["2015-01-01T16:00:00.000Z",
-                  "2015-01-01T17:00:00.000Z",
-                  "2015-01-01T18:00:00.000Z",
-                  "2015-01-01T19:00:00.000Z",
-                  "2015-01-01T20:00:00.000Z",
-                  "2015-01-01T21:00:00.000Z",
-                  "2015-01-01T22:00:00.000Z",
-                  "2015-01-01T23:00:00.000Z",
-                  "2015-01-02T00:00:00.000Z",
-                  "2015-01-02T01:00:00.000Z",
-                  "2015-01-02T02:00:00.000Z"].map { |i| Time.iso8601(i) }
-      actual = queue.size.times.map { queue.pop.get("custom_time").time }
-      expect(actual).to eq(expected)
-      plugin.stop
+      let(:msecs) { [111, 122, 233, 244, 355, 366, 477, 488, 599, 611, 722] }
 
-      plugin.run(queue)
-      expect(queue.size).to eq(0)
-      db[:test_table].insert(:num => 11, :custom_time => "2015-01-01 11:00:00", :created_at => Time.now.utc)
-      db[:test_table].insert(:num => 12, :custom_time => "2015-01-01 21:00:00", :created_at => Time.now.utc)
-      plugin.run(queue)
-      expect(queue.size).to eq(1)
-      event = queue.pop
-      expect(event.get("num")).to eq(12)
-      expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:00:00.000Z"))
-      p settings
+      it "should convert the time to reflect the timezone " do
+        # Sequel only does the *correct* timezone calc on a DateTime instance
+        last_run_value = DateTime.iso8601("2000-01-01T00:00:00.987Z")
+        File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+
+        hours.each_with_index do |i, j|
+          db[:test1_table].insert(:num => i, :custom_time => "2015-01-01 #{i}:00:00.#{msecs[j]}", :created_at => Time.now.utc)
+        end
+
+        plugin.register
+
+        plugin.run(queue)
+        expected = hours.map.with_index {|hour, i| Time.iso8601("2015-01-01T06:00:00.000Z") + (hour * 3600 + (msecs[i] / 1000.0)) }
+        actual = queue.size.times.map { queue.pop.get("custom_time").time }
+        expect(actual).to eq(expected)
+        plugin.stop
+        last_run_value = YAML.load(File.read(settings["last_run_metadata_path"]))
+        expect(last_run_value).to be_a(DateTime)
+        expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 02:00:00.722000000 +00:00")
+
+        plugin.run(queue)
+        plugin.stop
+        db[:test1_table].insert(:num => 11, :custom_time => "2015-01-01 11:00:00.099", :created_at => Time.now.utc)
+        db[:test1_table].insert(:num => 12, :custom_time => "2015-01-01 21:00:00.811", :created_at => Time.now.utc)
+        expect(queue.size).to eq(0)
+        plugin.run(queue)
+        expect(queue.size).to eq(1)
+        event = queue.pop
+        plugin.stop
+        expect(event.get("num")).to eq(12)
+        expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:00:00.811Z"))
+        last_run_value = YAML.load(File.read(settings["last_run_metadata_path"]))
+        expect(last_run_value).to be_a(DateTime)
+        # verify that sub-seconds are recorded to the file
+        expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 03:00:00.811000000 +00:00")
+      end
     end
   end
 
@@ -479,16 +564,16 @@ describe LogStash::Inputs::Jdbc do
       test_table = db[:test_table]
 
       plugin.run(queue)
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(0)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(0)
       test_table.insert(:num => nums[0], :created_at => Time.now.utc)
       test_table.insert(:num => nums[1], :created_at => Time.now.utc)
       plugin.run(queue)
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(20)
       test_table.insert(:num => nums[2], :created_at => Time.now.utc)
       test_table.insert(:num => nums[3], :created_at => Time.now.utc)
       test_table.insert(:num => nums[4], :created_at => Time.now.utc)
       plugin.run(queue)
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(50)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(50)
     end
   end
 
@@ -522,17 +607,17 @@ describe LogStash::Inputs::Jdbc do
       test_table = db[:test_table]
 
       plugin.run(queue)
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(Time.parse("1970-01-01 00:00:00.000000000 +0000"))
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(Time.parse("1970-01-01 00:00:00.000000000 +0000"))
       test_table.insert(:num => nums[0], :created_at => Time.now.utc, :custom_time => times[0])
       test_table.insert(:num => nums[1], :created_at => Time.now.utc, :custom_time => times[1])
       plugin.run(queue)
-      expect(plugin.instance_variable_get("@sql_last_value").class).to eq(Time.parse(times[0]).class)
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(Time.parse(times[1]))
+      expect(plugin.instance_variable_get("@value_tracker").value.class).to eq(Time.parse(times[0]).class)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(Time.parse(times[1]))
       test_table.insert(:num => nums[2], :created_at => Time.now.utc, :custom_time => times[2])
       test_table.insert(:num => nums[3], :created_at => Time.now.utc, :custom_time => times[3])
       test_table.insert(:num => nums[4], :created_at => Time.now.utc, :custom_time => times[4])
       plugin.run(queue)
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(Time.parse(times[4]))
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(Time.parse(times[4]))
     end
   end
 
@@ -566,19 +651,19 @@ describe LogStash::Inputs::Jdbc do
       test_table = db[:test_table]
 
       plugin.run(queue)
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(20)
       expect(queue.length).to eq(0) # Shouldn't grab anything here.
       test_table.insert(:num => nums[0], :created_at => Time.now.utc)
       test_table.insert(:num => nums[1], :created_at => Time.now.utc)
       plugin.run(queue)
       expect(queue.length).to eq(0) # Shouldn't grab anything here either.
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(20)
       test_table.insert(:num => nums[2], :created_at => Time.now.utc)
       test_table.insert(:num => nums[3], :created_at => Time.now.utc)
       test_table.insert(:num => nums[4], :created_at => Time.now.utc)
       plugin.run(queue)
       expect(queue.length).to eq(3) # Only values greater than 20 should be grabbed.
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(50)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(50)
     end
   end
 
@@ -612,19 +697,19 @@ describe LogStash::Inputs::Jdbc do
       test_table = db[:test_table]
 
       plugin.run(queue)
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(20)
       expect(queue.length).to eq(0) # Shouldn't grab anything here.
       test_table.insert(:num => nums[0], :created_at => Time.now.utc)
       test_table.insert(:num => nums[1], :created_at => Time.now.utc)
       plugin.run(queue)
       expect(queue.length).to eq(0) # Shouldn't grab anything here either.
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(20)
       test_table.insert(:num => nums[2], :created_at => Time.now.utc)
       test_table.insert(:num => nums[3], :created_at => Time.now.utc)
       test_table.insert(:num => nums[4], :created_at => Time.now.utc)
       plugin.run(queue)
       expect(queue.length).to eq(3) # Only values greater than 20 should be grabbed.
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(20)
       expect(plugin.instance_variable_get("@tracking_column_warning_sent")).to eq(true)
     end
   end
@@ -650,7 +735,7 @@ describe LogStash::Inputs::Jdbc do
     it "should respect last run metadata" do
       plugin.run(queue)
 
-      expect(plugin.instance_variable_get("@sql_last_value")).to be > last_run_time
+      expect(plugin.instance_variable_get("@value_tracker").value).to be > last_run_time
     end
   end
 
@@ -677,7 +762,7 @@ describe LogStash::Inputs::Jdbc do
     it "metadata should equal last_run_value" do
       plugin.run(queue)
 
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(last_run_value)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(last_run_value)
     end
   end
 
@@ -701,7 +786,7 @@ describe LogStash::Inputs::Jdbc do
     it "should not respect last run metadata" do
       plugin.run(queue)
 
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(last_run_time)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(last_run_time)
     end
   end
 
@@ -728,7 +813,7 @@ describe LogStash::Inputs::Jdbc do
     it "metadata should still reflect last value" do
       plugin.run(queue)
 
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(last_run_value)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(last_run_value)
     end
   end
 
@@ -754,7 +839,7 @@ describe LogStash::Inputs::Jdbc do
     end
 
     it "should ignore last run metadata if :clean_run set to true" do
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(Time.at(0).utc)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(Time.at(0).utc)
     end
   end
 
@@ -782,7 +867,7 @@ describe LogStash::Inputs::Jdbc do
     end
 
     it "should ignore last run metadata if :clean_run set to true" do
-      expect(plugin.instance_variable_get("@sql_last_value")).to eq(0)
+      expect(plugin.instance_variable_get("@value_tracker").value).to eq(0)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 4.3.4
+  version: 4.3.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-14 00:00:00.000000000 Z
+date: 2018-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -172,6 +172,7 @@ files:
 - docs/index.asciidoc
 - lib/logstash/inputs/jdbc.rb
 - lib/logstash/plugin_mixins/jdbc.rb
+- lib/logstash/plugin_mixins/value_tracking.rb
 - logstash-input-jdbc.gemspec
 - spec/inputs/jdbc_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html