logstash-input-jdbc 2.1.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6fc9b06fd127404a35d42af320ad90b37f0ff642
-  data.tar.gz: 21798f6ed0c0620f054acc6abe0c207211fadc70
+  metadata.gz: bf7a5bf46a7c317d17c7e12169046979b19242c1
+  data.tar.gz: 26fe05b541b4a84b764e08d6f9527bda90329639
 SHA512:
-  metadata.gz: bc1d4d90e171555d00334ab59ef67699bdf55e714557ef565a9b47d870b20d233f23fe7f0a48ed0025396ab822a9e18971d7eed1692027eac4971ced58f1ffb6
-  data.tar.gz: aeeb2afa34a2767d035c20e71c0efdd709f9df48988954761d064e542def704ba7c4ec8e52248187eb8df8847f6db1bfb5c60f081db2b1bd8709f6bb908210f3
+  metadata.gz: 955833f1f3abe24e373a193cac7e34d27de4ba0245cfb7210955e45a4a650c512ecd8e1f2044de746d057b386123cb2bd2be5efaba705b9e1e581b8d2ee268cf
+  data.tar.gz: d36bc646854ef3409fe41055eca173eabcb9ea19913738d658fded3ae410c95f3169c9e9b1ff3ff8fc8a2532c9c48e779588e140543bc9a613a5606a9fcca6e2

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.0.0
+  - [#57](https://github.com/logstash-plugins/logstash-input-jdbc/issues/57) New feature: Allow tracking by a column value rather than by last run time.  **This is a breaking change**, as users may be required to change from using `sql_last_start` to use `sql_last_value` in their queries.  No other changes are required if you've been using time-based queries.  See the documentation if you wish to use an incremental column value to track updates to your tables.
+
 ## 2.1.1
   - [#44](https://github.com/logstash-plugins/logstash-input-jdbc/issues/44) add option to control the lowercase or not, of the column names.
 

data/lib/logstash/inputs/jdbc.rb

@@ -35,12 +35,12 @@ require "yaml" # persistence
 #
 # ==== State
 #
-# The plugin will persist the `sql_last_start` parameter in the form of a
-# metadata file stored in the configured `last_run_metadata_path`. Upon shutting down,
-# this file will be updated with the current value of `sql_last_start`. Next time
+# The plugin will persist the `sql_last_value` parameter in the form of a
+# metadata file stored in the configured `last_run_metadata_path`. Upon query execution,
+# this file will be updated with the current value of `sql_last_value`. Next time
 # the pipeline starts up, this value will be updated by reading from the file. If
-# `clean_run` is set to true, this value will be ignored and `sql_last_start` will be
-# set to Jan 1, 1970, as if no query has ever been executed.
+# `clean_run` is set to true, this value will be ignored and `sql_last_value` will be
+# set to Jan 1, 1970, or 0 if `use_column_value` is true, as if no query has ever been executed.
 #
 # ==== Dealing With Large Result-sets
 #
@@ -90,8 +90,9 @@ require "yaml" # persistence
 # Here is the list:
 #
 # |==========================================================
-# |sql_last_start | The last time a statement was executed. This is set to Thursday, 1 January 1970
-#  before any query is run, and updated accordingly after first query is run.
+# |sql_last_value | The value used to calculate which rows to query. Before any query is run,
+# this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and
+# `tracking_column` is set. It is updated accordingly after subsequent queries are run.
 # |==========================================================
 #
 class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
@@ -131,6 +132,12 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   # Path to file with last run time
   config :last_run_metadata_path, :validate => :string, :default => "#{ENV['HOME']}/.logstash_jdbc_last_run"
 
+  # Use an incremental column value rather than a timestamp
+  config :use_column_value, :validate => :boolean, :default => false
+
+  # If tracking column value rather than timestamp, the column whose value is to be tracked
+  config :tracking_column, :validate => :string
+
   # Whether the previous run state should be preserved
   config :clean_run, :validate => :boolean, :default => false
 
@@ -146,11 +153,18 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     require "rufus/scheduler"
     prepare_jdbc_connection
 
-    # load sql_last_start from file if exists
+    # Raise an error if @use_column_value is true, but no @tracking_column is set
+    if @use_column_value
+      if @tracking_column.nil?
+        raise(LogStash::ConfigurationError, "Must set :tracking_column if :use_column_value is true.")
+      end
+    end
+
+    # load sql_last_value from file if exists
     if @clean_run && File.exist?(@last_run_metadata_path)
       File.delete(@last_run_metadata_path)
     elsif File.exist?(@last_run_metadata_path)
-      @sql_last_start = YAML.load(File.read(@last_run_metadata_path))
+      @sql_last_value = YAML.load(File.read(@last_run_metadata_path))
     end
 
     unless @statement.nil? ^ @statement_filepath.nil?
@@ -165,22 +179,19 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
       @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
       @scheduler.cron @schedule do
         execute_query(queue)
+        update_state_file
       end
 
       @scheduler.join
     else
       execute_query(queue)
+      update_state_file
     end
   end # def run
 
   def stop
     @scheduler.stop if @scheduler
 
-    # update state file for next run
-    if @record_last_run
-      File.write(@last_run_metadata_path, YAML.dump(@sql_last_start))
-    end
-
     close_jdbc_connection
   end
 
@@ -188,11 +199,18 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
 
   def execute_query(queue)
     # update default parameters
-    @parameters['sql_last_start'] = @sql_last_start
+    @parameters['sql_last_value'] = @sql_last_value
     execute_statement(@statement, @parameters) do |row|
       event = LogStash::Event.new(row)
       decorate(event)
       queue << event
     end
   end
+
+  def update_state_file
+    if @record_last_run
+      File.write(@last_run_metadata_path, YAML.dump(@sql_last_value))
+    end
+  end
+
 end # class LogStash::Inputs::Jdbc

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -161,7 +161,11 @@ module LogStash::PluginMixins::Jdbc
     else
       @database.identifier_output_method = :to_s
     end
-    @sql_last_start = Time.at(0).utc
+    if @use_column_value
+      @sql_last_value = 0
+    else
+      @sql_last_value = Time.at(0).utc
+    end
   end # def prepare_jdbc_connection
 
   public
@@ -175,27 +179,47 @@ module LogStash::PluginMixins::Jdbc
     begin
       parameters = symbolized_params(parameters)
       query = @database[statement, parameters]
+      sql_last_value = @use_column_value ? @sql_last_value : Time.now.utc
+      @tracking_column_warning_sent = false
       @logger.debug? and @logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters, :count => query.count)
-      @sql_last_start = Time.now.utc
 
       if @jdbc_paging_enabled
         query.each_page(@jdbc_page_size) do |paged_dataset|
           paged_dataset.each do |row|
+            sql_last_value = get_column_value(row) if @use_column_value
             yield extract_values_from(row)
           end
         end
       else
         query.each do |row|
+          sql_last_value = get_column_value(row) if @use_column_value
           yield extract_values_from(row)
         end
       end
       success = true
     rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError => e
       @logger.warn("Exception when executing JDBC query", :exception => e)
+    else
+      @sql_last_value = sql_last_value
     end
     return success
   end
 
+  public
+  def get_column_value(row)
+    if !row.has_key?(@tracking_column.to_sym)
+      if !@tracking_column_warning_sent
+        @logger.warn("tracking_column not found in dataset.", :tracking_column => @tracking_column)
+        @tracking_column_warning_sent = true
+      end
+      # If we can't find the tracking column, return the current value in the ivar
+      @sql_last_value
+    else
+      # Otherwise send the updated tracking column
+      row[@tracking_column.to_sym]
+    end
+  end
+
   # Symbolize parameters keys to use with Sequel
   private
   def symbolized_params(parameters)

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '2.1.1'
+  s.version         = '3.0.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example input streams a string at a definable interval."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -161,6 +161,40 @@ describe LogStash::Inputs::Jdbc do
 
   end
 
+  context "when scheduling and previous runs are to be preserved" do
+    let(:settings) do
+      {
+        "statement" => "SELECT 1 as num_param FROM SYSIBM.SYSDUMMY1",
+        "schedule" => "* * * * * UTC",
+        "last_run_metadata_path" => Stud::Temporary.pathname
+      }
+    end
+
+    let(:last_run_time) { Time.at(1).utc }
+
+    before do
+      plugin.register
+    end
+
+    it "should flush previous run metadata per query execution" do
+      Timecop.travel(Time.new(2000))
+      Timecop.scale(60)
+      runner = Thread.new do
+        plugin.run(queue)
+      end
+      sleep 1
+      for i in 0..1
+        sleep 1
+        updated_last_run = YAML.load(File.read(settings["last_run_metadata_path"]))
+        expect(updated_last_run).to be > last_run_time
+        last_run_time = updated_last_run
+      end
+
+      plugin.stop
+    end
+
+  end
+
   context "when iterating result-set via paging" do
 
     let(:settings) do
@@ -295,7 +329,7 @@ describe LogStash::Inputs::Jdbc do
 
   context "when iteratively running plugin#run" do
     let(:settings) do
-      {"statement" => "SELECT num, created_at FROM test_table WHERE created_at > :sql_last_start"}
+      {"statement" => "SELECT num, created_at FROM test_table WHERE created_at > :sql_last_value"}
     end
 
     let(:nums) { [10, 20, 30, 40, 50] }
@@ -329,14 +363,148 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
-  context "when previous runs are to be respected" do
+  context "when iteratively running plugin#run with tracking_column" do
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"
+      }
+    end
 
     let(:settings) do
-      { "statement" => "SELECT * FROM test_table",
+      { "statement" => "SELECT num, created_at FROM test_table WHERE num > :sql_last_value",
+        "use_column_value" => true,
+        "tracking_column" => "num",
         "last_run_metadata_path" => Stud::Temporary.pathname }
     end
 
-    let(:last_run_time) { Time.at(1).utc }
+    let(:nums) { [10, 20, 30, 40, 50] }
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should successfully update sql_last_value" do
+      test_table = db[:test_table]
+
+      plugin.run(queue)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(0)
+      test_table.insert(:num => nums[0], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[1], :created_at => Time.now.utc)
+      plugin.run(queue)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      test_table.insert(:num => nums[2], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[3], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[4], :created_at => Time.now.utc)
+      plugin.run(queue)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(50)
+    end
+  end
+
+  context "when iteratively running plugin#run with tracking_column and stored metadata" do
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"
+      }
+    end
+
+    let(:settings) do
+      { "statement" => "SELECT num, created_at FROM test_table WHERE num > :sql_last_value",
+        "use_column_value" => true,
+        "tracking_column" => "num",
+        "last_run_metadata_path" => Stud::Temporary.pathname }
+    end
+
+    let(:nums) { [10, 20, 30, 40, 50] }
+    let(:last_run_value) { 20 }
+
+    before do
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should successfully update sql_last_value and only add appropriate events" do
+      test_table = db[:test_table]
+
+      plugin.run(queue)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(queue.length).to eq(0) # Shouldn't grab anything here.
+      test_table.insert(:num => nums[0], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[1], :created_at => Time.now.utc)
+      plugin.run(queue)
+      expect(queue.length).to eq(0) # Shouldn't grab anything here either.
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      test_table.insert(:num => nums[2], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[3], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[4], :created_at => Time.now.utc)
+      plugin.run(queue)
+      expect(queue.length).to eq(3) # Only values greater than 20 should be grabbed.
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(50)
+    end
+  end
+
+  context "when iteratively running plugin#run with BAD tracking_column and stored metadata" do
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"
+      }
+    end
+
+    let(:settings) do
+      { "statement" => "SELECT num, created_at FROM test_table WHERE num > :sql_last_value",
+        "use_column_value" => true,
+        "tracking_column" => "not_num",
+        "last_run_metadata_path" => Stud::Temporary.pathname }
+    end
+
+    let(:nums) { [10, 20, 30, 40, 50] }
+    let(:last_run_value) { 20 }
+
+    before do
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should send a warning and not update sql_last_value" do
+      test_table = db[:test_table]
+
+      plugin.run(queue)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(queue.length).to eq(0) # Shouldn't grab anything here.
+      test_table.insert(:num => nums[0], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[1], :created_at => Time.now.utc)
+      plugin.run(queue)
+      expect(queue.length).to eq(0) # Shouldn't grab anything here either.
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      test_table.insert(:num => nums[2], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[3], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[4], :created_at => Time.now.utc)
+      plugin.run(queue)
+      expect(queue.length).to eq(3) # Only values greater than 20 should be grabbed.
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(20)
+      expect(plugin.instance_variable_get("@tracking_column_warning_sent")).to eq(true)
+    end
+  end
+
+  context "when previous runs are to be respected upon successful query execution (by time)" do
+
+    let(:settings) do
+      { "statement" => "SELECT 1 as num_param FROM SYSIBM.SYSDUMMY1",
+        "last_run_metadata_path" => Stud::Temporary.pathname }
+    end
+
+    let(:last_run_time) { Time.now.utc }
 
     before do
       File.write(settings["last_run_metadata_path"], YAML.dump(last_run_time))
@@ -348,11 +516,91 @@ describe LogStash::Inputs::Jdbc do
     end
 
     it "should respect last run metadata" do
-      expect(plugin.instance_variable_get("@sql_last_start")).to eq(last_run_time)
+      plugin.run(queue)
+
+      expect(plugin.instance_variable_get("@sql_last_value")).to be > last_run_time
+    end
+  end
+
+  context "when previous runs are to be respected upon successful query execution (by column)" do
+
+    let(:settings) do
+      { "statement" => "SELECT 1 as num_param FROM SYSIBM.SYSDUMMY1",
+        "use_column_value" => true,
+        "tracking_column" => "num_param",
+        "last_run_metadata_path" => Stud::Temporary.pathname }
+    end
+
+    let(:last_run_value) { 1 }
+
+    before do
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "metadata should equal last_run_value" do
+      plugin.run(queue)
+
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(last_run_value)
+    end
+  end
+
+  context "when previous runs are to be respected upon query failure (by time)" do
+    let(:settings) do
+      { "statement" => "SELECT col from non_existent_table",
+        "last_run_metadata_path" => Stud::Temporary.pathname }
+    end
+
+    let(:last_run_time) { Time.now.utc }
+
+    before do
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_time))
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should not respect last run metadata" do
+      plugin.run(queue)
+
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(last_run_time)
     end
   end
 
-  context "when doing a clean run" do
+  context "when previous runs are to be respected upon query failure (by column)" do
+    let(:settings) do
+      { "statement" => "SELECT col from non_existent_table",
+        "use_column_value" => true,
+        "tracking_column" => "num_param",
+        "last_run_metadata_path" => Stud::Temporary.pathname
+      }
+    end
+
+    let(:last_run_value) { 1 }
+
+    before do
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "metadata should still reflect last value" do
+      plugin.run(queue)
+
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(last_run_value)
+    end
+  end
+
+  context "when doing a clean run (by time)" do
 
     let(:settings) do
       {
@@ -374,10 +622,39 @@ describe LogStash::Inputs::Jdbc do
     end
 
     it "should ignore last run metadata if :clean_run set to true" do
-      expect(plugin.instance_variable_get("@sql_last_start")).to eq(Time.at(0).utc)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(Time.at(0).utc)
     end
   end
 
+  context "when doing a clean run (by value)" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * FROM test_table",
+        "last_run_metadata_path" => Stud::Temporary.pathname,
+        "use_column_value" => true,
+        "tracking_column" => "num_param",
+        "clean_run" => true
+      }
+    end
+
+    let(:last_run_value) { 1000 }
+
+    before do
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should ignore last run metadata if :clean_run set to true" do
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(0)
+    end
+  end
+
+
   context "when state is not to be persisted" do
     let(:settings) do
       {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-17 00:00:00.000000000 Z
+date: 2016-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core