logstash-integration-jdbc 5.0.0 → 5.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5e61cdfeea13d5b4fe6183e2d3877e0de3b21d5542d7d4b6c53bee0d26735c82
-  data.tar.gz: 4f9e93b0d787e0e0fe69bebdaa1676c0b4fe0681a5e65e58661a49807e4e8e33
+  metadata.gz: a71840d1b072e13fc4ddcc96cee6a6f68e8c12608167a138858c147c11795509
+  data.tar.gz: e782665b9e41de726ccf59e6110d7f2f77fc49127ce4bfe6c5bf16683ab2ffb2
 SHA512:
-  metadata.gz: 47c7befa425f46f45c2ba621742acd8d2cc4aaed39f8f63a8d457333f33cebf3b443f6dfaeeb7baddf3a5d23ab87c067175c4c12a6667f1d2a88baafa97c6488
-  data.tar.gz: f2273443e0326dacf3809c79d01ced56c383c2632821160ac6c203d0b3479ed63df7a5ce6fe1711f2c7630aa2510e53b492248d3474ef651873d77589252fd1b
+  metadata.gz: 9166b1b3b4e0d461da084d78e5f69ad12ea07d33d61331520bfe755f920df5c406e414f9c60d2209e2432237c6842b1a627b2a852a91cfc250fa4fee0b934f70
+  data.tar.gz: 906d918fd928c2263ec6f1497d7995e7c0f6d9caa8044e23dcc2f8cb020c44c3c83372ed1cf808f3ee86aea33529d7b4378df778fa908ca2c5a97814e45df817

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 5.0.1
+  - Fixed tracking_column regression with Postgresql Numeric types [#17](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/17)
+  - Fixed driver loading when file not accessible [#15](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/15)
+
 ## 5.0.0
   - Initial Release of JDBC Integration Plugin, incorporating [logstash-input-jdbc](https://github.com/logstash-plugins/logstash-input-jdbc), [logstash-filter-jdbc_streaming](https://github.com/logstash-plugins/logstash-filter-jdbc_streaming) and
     [logstash-filter-jdbc_static](https://github.com/logstash-plugins/logstash-filter-jdbc_static)

data/docs/input-jdbc.asciidoc

@@ -339,6 +339,7 @@ required you can pass them separated by a comma.
 NOTE: If not provided, Plugin will look for the driver class in the Logstash Java classpath. Additionally, if the library
  does not appear to be being loaded correctly via this setting, placing the relevant jar(s) in the Logstash Java
  classpath rather than via this setting may help.
+ Please also make sure the path is readable by the Logstash process (e.g. `logstash` user when running as a service).
 
 [id="plugins-{type}s-{plugin}-jdbc_fetch_size"]
 ===== `jdbc_fetch_size`

data/lib/logstash/inputs/jdbc.rb

@@ -270,6 +270,7 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   end
 
   def run(queue)
+    load_driver
     if @schedule
       @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
       @scheduler.cron @schedule do

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -139,47 +139,58 @@ module LogStash  module PluginMixins module Jdbc
 
     private
 
+    def load_driver
+      if @drivers_loaded.false?
+        require "java"
+        require "sequel"
+        require "sequel/adapters/jdbc"
+
+        load_driver_jars
+        begin
+          Sequel::JDBC.load_driver(@jdbc_driver_class)
+        rescue Sequel::AdapterNotFound => e # Sequel::AdapterNotFound, "#{@jdbc_driver_class} not loaded"
+          # fix this !!!
+          message = if jdbc_driver_library_set?
+                      "Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
+                    else
+                      ":jdbc_driver_library is not set, are you sure you included " +
+                          "the proper driver client libraries in your classpath?"
+                    end
+          raise LogStash::PluginLoadingError, "#{e}. #{message}"
+        end
+        @drivers_loaded.make_true
+      end
+    end
+
     def load_driver_jars
-      unless @jdbc_driver_library.nil? || @jdbc_driver_library.empty?
+      if jdbc_driver_library_set?
         @jdbc_driver_library.split(",").each do |driver_jar|
+          @logger.debug("loading #{driver_jar}")
+          # load 'driver.jar' is different than load 'some.rb' as it only causes the file to be added to
+          # JRuby's class-loader lookup (class) path - won't raise a LoadError when file is not readable
+          unless FileTest.readable?(driver_jar)
+            raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, " +
+                "file not readable (please check user and group permissions for the path)"
+          end
           begin
-            @logger.debug("loading #{driver_jar}")
-            # Use https://github.com/jruby/jruby/wiki/CallingJavaFromJRuby#from-jar-files to make classes from jar
-            # available
             require driver_jar
           rescue LoadError => e
             raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
+          rescue StandardError => e
+            raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e}"
           end
         end
       end
     end
 
-    private
-    def open_jdbc_connection
-      require "java"
-      require "sequel"
-      require "sequel/adapters/jdbc"
+    def jdbc_driver_library_set?
+      !@jdbc_driver_library.nil? && !@jdbc_driver_library.empty?
+    end
 
+    def open_jdbc_connection
+      # at this point driver is already loaded
       Sequel.application_timezone = @plugin_timezone.to_sym
-      if @drivers_loaded.false?
-        begin
-          load_driver_jars
-          Sequel::JDBC.load_driver(@jdbc_driver_class)
-        rescue LogStash::Error => e
-          # raised in load_drivers, e.cause should be the caught Java exceptions
-          raise LogStash::PluginLoadingError, "#{e.message} and #{e.cause.message}"
-        rescue Sequel::AdapterNotFound => e
-          # fix this !!!
-          message = if @jdbc_driver_library.nil?
-            ":jdbc_driver_library is not set, are you sure you included
-                      the proper driver client libraries in your classpath?"
-          else
-            "Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
-          end
-          raise LogStash::PluginLoadingError, "#{e}. #{message}"
-        end
-        @drivers_loaded.make_true
-      end
+
       @database = jdbc_connect()
       @database.extension(:pagination)
       if @jdbc_default_timezone

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -62,7 +62,7 @@ module LogStash module PluginMixins module Jdbc
 
   class NumericValueTracker < ValueTracking
     def set_initial
-      common_set_initial(:gcd, 0)
+      common_set_initial(:abs, 0)
     end
 
     def set_value(value)

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.0.0'
+  s.version         = '5.0.1'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -28,10 +28,11 @@ describe LogStash::Inputs::Jdbc do
       # before body
       Jdbc::Derby.load_driver
       db.create_table :test_table do
-        DateTime :created_at
-        Integer  :num
-        String   :string
-        DateTime :custom_time
+        DateTime     :created_at
+        BigDecimal   :big_num
+        Integer      :num
+        String       :string
+        DateTime     :custom_time
       end
       db << "CREATE TABLE types_table (num INTEGER, string VARCHAR(255), started_at DATE, custom_time TIMESTAMP, ranking DECIMAL(16,6))"
       db << "CREATE TABLE test1_table (num INTEGER, string VARCHAR(255), custom_time TIMESTAMP, created_at TIMESTAMP)"
@@ -545,6 +546,54 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "Test Sql Last Value with a BigDecimal `sqlLastValue`" do
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"
+      }
+    end
+
+    let(:settings) do
+      { "statement" => "SELECT big_num, created_at FROM test_table WHERE big_num > :sql_last_value",
+        "use_column_value" => true,
+        "tracking_column" => "big_num",
+        "last_run_metadata_path" => Stud::Temporary.pathname }
+    end
+
+    let(:nums) { [BigDecimal.new(10), BigDecimal.new(20), BigDecimal.new(30), BigDecimal.new(40), BigDecimal.new(50)] }
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should only pick up new values" do
+      test_table = db[:test_table]
+
+      plugin.run(queue)
+      expect(queue.size).to eq(0)
+      test_table.insert(:big_num => nums[0], :created_at => Time.now.utc)
+      test_table.insert(:big_num => nums[1], :created_at => Time.now.utc)
+      queue = []
+      # Stop and restart the plugin to have it read saved value of `sql_last_value`
+      plugin.stop
+      plugin.register
+      plugin.run(queue)
+      expect(queue.size).to eq(2)
+      test_table.insert(:big_num => nums[2], :created_at => Time.now.utc)
+      test_table.insert(:big_num => nums[3], :created_at => Time.now.utc)
+      test_table.insert(:big_num => nums[4], :created_at => Time.now.utc)
+      queue = []
+      plugin.stop
+      plugin.register
+      plugin.run(queue)
+      expect(queue.size).to eq(3)
+    end
+  end
+
   context "when iteratively running plugin#run with timestamp tracking column with column value" do
     let(:mixin_settings) do
       { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
@@ -1291,6 +1340,38 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "when an unreadable jdbc_driver_path entry is present" do
+    let(:driver_jar_path) do
+      jar_file = $CLASSPATH.find { |name| name.index(Jdbc::Derby.driver_jar) }
+      raise "derby jar not found on class-path" unless jar_file
+      jar_file.sub('file:', '')
+    end
+
+    let(:invalid_driver_jar_path) do
+      path = File.join(Dir.mktmpdir, File.basename(driver_jar_path))
+      FileUtils.cp driver_jar_path, path
+      FileUtils.chmod "u=x,go=", path
+      path
+    end
+
+    let(:settings) do
+      { "statement" => "SELECT * from types_table", "jdbc_driver_library" => invalid_driver_jar_path }
+    end
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "raise a loading error" do
+      expect { plugin.run(queue) }.
+          to raise_error(LogStash::PluginLoadingError, /unable to load .*? from :jdbc_driver_library, file not readable/)
+    end
+  end
+
   context "when using prepared statements" do
     let(:last_run_value) { 250 }
     let(:expected_queue_size) { 100 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 5.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-01-09 00:00:00.000000000 Z
+date: 2020-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement