logstash-input-jdbc 4.1.1 → 4.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7757853b90560fd7131dc285e37a5ac28a21eaf3
-  data.tar.gz: b6437464d388b823fae8a83bc87d744335ba702f
+  metadata.gz: a78d72e92aa01bf8891190bf7f03fb389f26b59f
+  data.tar.gz: 014325a08ad778dcc57aafeca4c45978be18abf7
 SHA512:
-  metadata.gz: 255dfb0c17325a467a418a2a74c3d1a003f52cb9a845fe20aaa95f81cd5c1724f105943e5091de6ba470f21b85ff0a59fe27665b95edf31d7d20f0d532210607
-  data.tar.gz: c39dfd89ae3700a4e6259b8fe42b86e134a1f66fd56aa8f5d8714524d1899cb247b960ae8447c8caedd698ea20844c8b03ea9e7cb58e320b531f60b1f1551a0c
+  metadata.gz: 8d4841251922f7b763a4972e5cd47b8b499f419b4c4771656dea5a769c0bc9484eda5591073b37b34ba46c262b27dfef5b217e8a559aa731c80886bc29f9f9a8
+  data.tar.gz: c9913e1d5b4b686d085d9875b38a7a27837ba8a327fa3cd4ce326bf5ffab5e24d630cd3a3184e5aea4a6f8a871f338d0a8d249db78295c8e8933d0c645e4cc72

data/CHANGELOG.md

@@ -1,36 +1,47 @@
+## 4.1.2
+ - [internal] Removed docker dependencies for testing
+
 ## 4.1.1
-  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
+ - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 
 ## 4.1.0
-  - Add an option to select the encoding data should be transform from,
-    this will make sure all strings read from the jdbc connector are
-    noremalized to be UTF-8 so no causing issues with later filters in LS.
+ - Add an option to select the encoding data should be transform from,
+   this will make sure all strings read from the jdbc connector are
+   noremalized to be UTF-8 so no causing issues with later filters in LS.
+
 ## 4.0.1
-  - Republish all the gems under jruby.
+ - Republish all the gems under jruby.
+
 ## 4.0.0
-  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+
 # 3.0.3
-  - Added feature to read password from external file (#120)
+ - Added feature to read password from external file (#120)
+
 # 3.0.2
-  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+
 # 3.0.1
-  - New dependency requirements for logstash-core for the 5.0 release
+ - New dependency requirements for logstash-core for the 5.0 release
+ - feature: Added configurable support for retrying database connection
+   failures.
+
 ## 3.0.0
-  - [#57](https://github.com/logstash-plugins/logstash-input-jdbc/issues/57) New feature: Allow tracking by a column value rather than by last run time.  **This is a breaking change**, as users may be required to change from using `sql_last_start` to use `sql_last_value` in their queries.  No other changes are required if you've been using time-based queries.  See the documentation if you wish to use an incremental column value to track updates to your tables.
+ - [#57](https://github.com/logstash-plugins/logstash-input-jdbc/issues/57) New feature: Allow tracking by a column value rather than by last run time.  **This is a breaking change**, as users may be required to change from using `sql_last_start` to use `sql_last_value` in their queries.  No other changes are required if you've been using time-based queries.  See the documentation if you wish to use an incremental column value to track updates to your tables.
 
 ## 2.1.1
-  - [#44](https://github.com/logstash-plugins/logstash-input-jdbc/issues/44) add option to control the lowercase or not, of the column names.
+ - [#44](https://github.com/logstash-plugins/logstash-input-jdbc/issues/44) add option to control the lowercase or not, of the column names.
 
 ## 2.1.0
-  - [#85](https://github.com/logstash-plugins/logstash-input-jdbc/issues/85) make the jdbc_driver_library accept a list of elements separated by commas as in some situations we might need to load more than one jar/lib.
-  - [#89](https://github.com/logstash-plugins/logstash-input-jdbc/issues/89) Set application timezone for cases where time fields in data have no timezone.
+ - [#85](https://github.com/logstash-plugins/logstash-input-jdbc/issues/85) make the jdbc_driver_library accept a list of elements separated by commas as in some situations we might need to load more than one jar/lib.
+ - [#89](https://github.com/logstash-plugins/logstash-input-jdbc/issues/89) Set application timezone for cases where time fields in data have no timezone.
 
 ## 2.0.5
-  - [#77](https://github.com/logstash-plugins/logstash-input-jdbc/issues/77) Time represented as RubyTime and not as Logstash::Timestamp
+ - [#77](https://github.com/logstash-plugins/logstash-input-jdbc/issues/77) Time represented as RubyTime and not as Logstash::Timestamp
 
 ## 2.0.4
-  - [#70](https://github.com/logstash-plugins/logstash-input-jdbc/pull/70) prevents multiple queries from being run at the same time
-  - [#69](https://github.com/logstash-plugins/logstash-input-jdbc/pull/69) pass password as string to Sequel
+ - [#70](https://github.com/logstash-plugins/logstash-input-jdbc/pull/70) prevents multiple queries from being run at the same time
+ - [#69](https://github.com/logstash-plugins/logstash-input-jdbc/pull/69) pass password as string to Sequel
 
 ## 2.0.3
  - Added ability to configure timeout
@@ -41,5 +52,5 @@
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 
-* 1.0.0
-  - Initial release
+## 1.0.0
+ - Initial release

data/lib/logstash/inputs/jdbc.rb

@@ -151,6 +151,9 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   # If tracking column value rather than timestamp, the column whose value is to be tracked
   config :tracking_column, :validate => :string
 
+  # Type of tracking column. Currently only "numeric" and "timestamp"
+  config :tracking_column_type, :validate => ['numeric', 'timestamp'], :default => 'numeric'
+
   # Whether the previous run state should be preserved
   config :clean_run, :validate => :boolean, :default => false
 
@@ -183,6 +186,7 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   public
 
   def register
+    @logger = self.logger
     require "rufus/scheduler"
     prepare_jdbc_connection
 

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -8,8 +8,6 @@ require "date"
 # for potential reuse in other plugins (input/output)
 module LogStash::PluginMixins::Jdbc
 
-  @logger = Cabin::Channel.get(LogStash)
-
   # This method is called when someone includes this module
   def self.included(base)
     # Add these methods to the 'base' given.
@@ -183,7 +181,12 @@ module LogStash::PluginMixins::Jdbc
       @database.identifier_output_method = :to_s
     end
     if @use_column_value
-      @sql_last_value = 0
+      case @tracking_column_type
+        when "numeric"
+          @sql_last_value = 0
+        when "timestamp"
+          @sql_last_value = Time.at(0).utc
+      end
     else
       @sql_last_value = Time.at(0).utc
     end
@@ -208,12 +211,18 @@ module LogStash::PluginMixins::Jdbc
         query.each_page(@jdbc_page_size) do |paged_dataset|
           paged_dataset.each do |row|
             sql_last_value = get_column_value(row) if @use_column_value
+            if @tracking_column_type=="timestamp" and @use_column_value and sql_last_value.is_a?(DateTime)
+              sql_last_value=Time.parse(sql_last_value.to_s) # Coerce the timestamp to a `Time`
+            end
             yield extract_values_from(row)
           end
         end
       else
         query.each do |row|
           sql_last_value = get_column_value(row) if @use_column_value
+          if @tracking_column_type=="timestamp" and @use_column_value and sql_last_value.is_a?(DateTime)
+            sql_last_value=Time.parse(sql_last_value.to_s) # Coerce the timestamp to a `Time`
+          end
           yield extract_values_from(row)
         end
       end

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '4.1.1'
+  s.version         = '4.1.2'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example input streams a string at a definable interval."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -28,5 +28,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'
   s.add_development_dependency 'jdbc-derby'
-  s.add_development_dependency 'docker-api'
 end

data/spec/inputs/jdbc_spec.rb

@@ -463,6 +463,50 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "when iteratively running plugin#run with timestamp tracking column with column value" do
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"
+      }
+    end
+
+    let(:settings) do
+      { "statement" => "SELECT num, created_at, custom_time FROM test_table WHERE custom_time > :sql_last_value",
+        "use_column_value" => true,
+        "tracking_column" => "custom_time",
+        "tracking_column_type" => "timestamp",
+        "last_run_metadata_path" => Stud::Temporary.pathname }
+    end
+
+    let(:nums) { [10, 20, 30, 40, 50] }
+    let(:times) {["2015-05-06 13:14:15","2015-05-07 13:14:15","2015-05-08 13:14:15","2015-05-09 13:14:15","2015-05-10 13:14:15"]}
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should successfully update sql_last_value" do
+      test_table = db[:test_table]
+
+      plugin.run(queue)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(Time.parse("1970-01-01 00:00:00.000000000 +0000"))
+      test_table.insert(:num => nums[0], :created_at => Time.now.utc, :custom_time => times[0])
+      test_table.insert(:num => nums[1], :created_at => Time.now.utc, :custom_time => times[1])
+      plugin.run(queue)
+      expect(plugin.instance_variable_get("@sql_last_value").class).to eq(Time.parse(times[0]).class)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(Time.parse(times[1]))
+      test_table.insert(:num => nums[2], :created_at => Time.now.utc, :custom_time => times[2])
+      test_table.insert(:num => nums[3], :created_at => Time.now.utc, :custom_time => times[3])
+      test_table.insert(:num => nums[4], :created_at => Time.now.utc, :custom_time => times[4])
+      plugin.run(queue)
+      expect(plugin.instance_variable_get("@sql_last_value")).to eq(Time.parse(times[4]))
+    end
+  end
+
   context "when iteratively running plugin#run with tracking_column and stored metadata" do
     let(:mixin_settings) do
       { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
@@ -821,8 +865,7 @@ describe LogStash::Inputs::Jdbc do
     let(:num_rows) { 5 }
 
     before do
-      plugin.instance_variable_set("@logger", logger)
-      allow(logger).to receive(:debug?)
+      allow(plugin.logger).to receive(:debug?)
       num_rows.times do
         db[:test_table].insert(:num => 1)
       end
@@ -834,10 +877,8 @@ describe LogStash::Inputs::Jdbc do
       plugin.stop
     end
 
-    let(:logger) { double("logger") }
-
-    it "should report the staments to logging" do
-      expect(logger).to receive(:debug).with(kind_of(String)).once
+    it "should report the statements to logging" do
+      expect(plugin.logger).to receive(:debug).once
       plugin.run(queue)
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.1.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -142,20 +142,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: docker-api
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -193,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This example input streams a string at a definable interval.