logstash-integration-jdbc 5.1.6 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef9180da50049c113170083c54826d75aadc91883fdca644bd82b5802ccd7234
-  data.tar.gz: 55b6adf668a1aac55c7a5883ae02de29431bcdf1e58d2177ab6004524bd99780
+  metadata.gz: a41fe10947372ac69f9db7a66dd1c285000f27bba0cb77ef7ec62629821fa5b3
+  data.tar.gz: 74b411ebe9f448259da6751def456535bf6819f84c80027a629354140fc5c779
 SHA512:
-  metadata.gz: 22e48f5ac21bde7821142cc77accd458dd695a8895941bed7d7a495459d06663c2166c94c91f9592f1621f119a9c796c186401542a8085f01abd6544b3345376
-  data.tar.gz: 3f0a613c6bd1d4a9c8b272ec6b344bc0f791f5a137f25adbee547fbece9d7a5a01028f125641255186266d47714bb47e499325c21700d865c4cc9bf22dd529f2
+  metadata.gz: '029c86dfb0a4099979c77443c79edc7ebfb1cc3572d54c43c9864642aa018892cede9e749685846ad3451defabb1cea1cc770f3c4062afaa06690a67c0a1d498'
+  data.tar.gz: 3c60fa57ef6c77bee21be8c3eac9b12c5f32af0b9fa7b4874dd8503ce6926b87ff139a2293a2276156df0ae6ec20ce238ed467ee803bc510452ab2d732d090a7

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 5.2.0
+  - Added `jdbc_paging_mode` option to choose if use `explicit` pagination in statements and avoid the initial count 
+    query or use `auto` to delegate to the underlying library [#95](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/95)
+
+## 5.1.10
+  - Refactor: to explicit Java (driver) class name loading [#96](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/96),
+    the change is expected to provide a more robust fix for the driver loading issue [#83](https://github.com/logstash-plugins/logstash-integration-jdbc/issues/83).
+
+    NOTE: a fatal driver error will no longer keep reloading the pipeline and now leads to a system exit. 
+
+  - Fix: regression due returning the Java driver class [#98](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/98)
+
+## 5.1.9 (yanked)
+  - Refactor: to explicit Java (driver) class name loading [#96](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/96),
+    the change is expected to provide a more robust fix for the driver loading issue [#83](https://github.com/logstash-plugins/logstash-integration-jdbc/issues/83).
+
+## 5.1.8
+  - Fix the blocking pipeline reload and shutdown when connectivity issues happen [#85](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/85)
+
+## 5.1.7
+  - Normalize jdbc_driver_class loading to support any top-level java packages [#86](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/86)
+
 ## 5.1.6
   - Fix, serialize the JDBC driver loading steps to avoid concurrency issues [#84](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/84)
 

data/docs/input-jdbc.asciidoc

@@ -129,6 +129,9 @@ Here is the list:
 |sql_last_value | The value used to calculate which rows to query. Before any query is run,
 this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and
 `tracking_column` is set. It is updated accordingly after subsequent queries are run.
+|offset, size| Values used with manual paging mode to explicitly implement the paging.
+Supported only if <<plugins-{type}s-{plugin}-jdbc_paging_enabled>> is enabled and
+<<plugins-{type}s-{plugin}-jdbc_paging_mode>> has the `explicit` value.
 |==========================================================
 
 Example:
@@ -153,7 +156,7 @@ NOTE: Not all JDBC accessible technologies will support prepared statements.
 With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on.
 Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array.
 
-NOTE: Building count queries around a prepared statement is not supported at this time and because jdbc paging uses count queries under the hood, jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements.
+NOTE: Building count queries around a prepared statement is not supported at this time. Because jdbc paging uses count queries when `jdbc_paging_mode` has value `auto`，jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements.
 
 Example:
 [source,ruby]
@@ -193,6 +196,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-jdbc_fetch_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-jdbc_page_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-jdbc_paging_enabled>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-jdbc_paging_mode>> |<<string,string>>, one of `["auto", "explicit"]`|No
 | <<plugins-{type}s-{plugin}-jdbc_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-jdbc_password_filepath>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-jdbc_pool_timeout>> |<<number,number>>|No
@@ -373,6 +377,52 @@ result-set. The limit size is set with `jdbc_page_size`.
 
 Be aware that ordering is not guaranteed between queries.
 
+[id="plugins-{type}s-{plugin}-jdbc_paging_mode"]
+===== `jdbc_paging_mode`
+
+  * Value can be any of: `auto`, `explicit`
+  * Default value is `"auto"`
+
+Whether to use `explicit` or `auto` mode during the JDBC paging
+
+If `auto`, your statement will be automatically surrounded by a count query and subsequent multiple paged queries (with `LIMIT` statement, etc.).
+
+If `explicit`, multiple queries (without a count query ahead) will be performed with your statement, until no more rows are retrieved.
+You have to write your own paging conditions in your statement configuration.
+The `offset` and `size` parameters can be used in your statement (`size` equal to `jdbc_page_size`, and `offset` incremented by `size` for each query).
+When the number of rows returned by the query is not equal to `size`, SQL paging will be ended.
+Example:
+
+[source, ruby]
+------------------------------------------------------
+input {
+  jdbc {
+    statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value LIMIT :size OFFSET :offset",
+    jdbc_paging_enabled => true,
+    jdbc_paging_mode => "explicit",
+    jdbc_page_size => 100000
+  }
+}
+------------------------------------------------------
+
+[source, ruby]
+------------------------------------------------------
+input {
+  jdbc {
+    statement => "CALL fetch_my_data(:sql_last_value, :offset, :size)",
+    jdbc_paging_enabled => true,
+    jdbc_paging_mode => "explicit",
+    jdbc_page_size => 100000
+  }
+}
+------------------------------------------------------
+
+This mode can be considered in the following situations:
+
+. Performance issues encountered in default paging mode.
+. Your SQL statement is complex, so simply surrounding it with paging statements is not what you want.
+. Your statement is a stored procedure, and the actual paging statement is inside it.
+
 [id="plugins-{type}s-{plugin}-jdbc_password"]
 ===== `jdbc_password`
 

data/lib/logstash/plugin_mixins/jdbc/common.rb

@@ -1,9 +1,12 @@
+require 'jruby'
 
 module LogStash module PluginMixins module Jdbc
   module Common
 
     private
 
+    # NOTE: using the JRuby mechanism to load classes (through JavaSupport)
+    # makes the lock redundant although it does not hurt to have it around.
     DRIVERS_LOADING_LOCK = java.util.concurrent.locks.ReentrantLock.new()
 
     def complete_sequel_opts(defaults = {})
@@ -30,16 +33,16 @@ module LogStash module PluginMixins module Jdbc
       begin
         load_driver_jars
         begin
-          @driver_impl = Sequel::JDBC.load_driver(@jdbc_driver_class)
-        rescue Sequel::AdapterNotFound => e # Sequel::AdapterNotFound, "#{@jdbc_driver_class} not loaded"
-          # fix this !!!
+          @driver_impl = load_jdbc_driver_class
+        rescue => e # catch java.lang.ClassNotFoundException, potential errors
+          # (e.g. ExceptionInInitializerError or LinkageError) won't get caught
           message = if jdbc_driver_library_set?
                       "Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
                     else
                       ":jdbc_driver_library is not set, are you sure you included " +
                           "the proper driver client libraries in your classpath?"
                     end
-          raise LogStash::PluginLoadingError, "#{e}. #{message} #{e.backtrace}"
+          raise LogStash::PluginLoadingError, "#{e.inspect}. #{message}"
         end
       ensure
         DRIVERS_LOADING_LOCK.unlock()
@@ -71,5 +74,16 @@ module LogStash module PluginMixins module Jdbc
       !@jdbc_driver_library.nil? && !@jdbc_driver_library.empty?
     end
 
+    def load_jdbc_driver_class
+      # sub a potential: 'Java::org::my.Driver' to 'org.my.Driver'
+      klass = @jdbc_driver_class.gsub('::', '.').sub(/^Java\./, '')
+      # NOTE: JRuby's Java::JavaClass.for_name which considers the custom class-loader(s)
+      # in 9.3 the API changed and thus to avoid surprises we go down to the Java API :
+      klass = JRuby.runtime.getJavaSupport.loadJavaClass(klass) # throws ClassNotFoundException
+      # unfortunately we can not simply return the wrapped java.lang.Class instance as
+      # Sequel assumes to be able to do a `driver_class.new` which only works on the proxy,
+      org.jruby.javasupport.Java.getProxyClass(JRuby.runtime, klass)
+    end
+
   end
 end end end

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -55,6 +55,9 @@ module LogStash  module PluginMixins module Jdbc
       # Be aware that ordering is not guaranteed between queries.
       config :jdbc_paging_enabled, :validate => :boolean, :default => false
 
+      # Which pagination mode to use, automatic pagination or explicitly defined in the query.
+      config :jdbc_paging_mode, :validate => [ "auto", "explicit" ], :default => "auto"
+
       # JDBC page size
       config :jdbc_page_size, :validate => :number, :default => 100000
 
@@ -168,8 +171,6 @@ module LogStash  module PluginMixins module Jdbc
         @logger.warn("Failed test_connection with java.sql.SQLException.", :exception => e)
       rescue Sequel::DatabaseConnectionError => e
         @logger.warn("Failed test_connection.", :exception => e)
-        close_jdbc_connection
-
         #TODO return false and let the plugin raise a LogStash::ConfigurationError
         raise e
       end
@@ -208,9 +209,9 @@ module LogStash  module PluginMixins module Jdbc
     public
     def execute_statement
       success = false
-      @connection_lock.lock
-      open_jdbc_connection
       begin
+        @connection_lock.lock
+        open_jdbc_connection
         sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
         @tracking_column_warning_sent = false
         @statement_handler.perform_query(@database, @value_tracker.value, @jdbc_paging_enabled, @jdbc_page_size) do |row|

data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb

@@ -3,7 +3,15 @@
 module LogStash module PluginMixins module Jdbc
   class StatementHandler
     def self.build_statement_handler(plugin, logger)
-      klass = plugin.use_prepared_statements ? PreparedStatementHandler : NormalStatementHandler
+      if plugin.use_prepared_statements
+        klass = PreparedStatementHandler
+      else
+        if plugin.jdbc_paging_enabled && plugin.jdbc_paging_mode == "explicit"
+          klass = ExplicitPagingModeStatementHandler
+        else
+          klass = NormalStatementHandler
+        end
+      end
       klass.new(plugin, logger)
     end
 
@@ -27,7 +35,9 @@ module LogStash module PluginMixins module Jdbc
   class NormalStatementHandler < StatementHandler
     # Performs the query, respecting our pagination settings, yielding once per row of data
     # @param db [Sequel::Database]
-    # @param sql_last_value [Integet|DateTime|Time]
+    # @param sql_last_value [Integer|DateTime|Time]
+    # @param jdbc_paging_enabled [Boolean]
+    # @param jdbc_page_size [Integer]
     # @yieldparam row [Hash{Symbol=>Object}]
     def perform_query(db, sql_last_value, jdbc_paging_enabled, jdbc_page_size)
       query = build_query(db, sql_last_value)
@@ -67,6 +77,28 @@ module LogStash module PluginMixins module Jdbc
     end
   end
 
+  class ExplicitPagingModeStatementHandler < NormalStatementHandler
+    # Performs the query, respecting our pagination settings, yielding once per row of data
+    # @param db [Sequel::Database]
+    # @param sql_last_value [Integer|DateTime|Time]
+    # @param jdbc_paging_enabled [Boolean]
+    # @param jdbc_page_size [Integer]
+    # @yieldparam row [Hash{Symbol=>Object}]
+    def perform_query(db, sql_last_value, jdbc_paging_enabled, jdbc_page_size)
+      query = build_query(db, sql_last_value)
+      offset = 0
+      loop do
+        rows_in_page = 0
+        query.with_sql(query.sql, offset: offset, size: jdbc_page_size).each do |row|
+          yield row
+          rows_in_page += 1
+        end
+        break unless rows_in_page == jdbc_page_size
+        offset += jdbc_page_size
+      end
+    end
+  end
+
   class PreparedStatementHandler < StatementHandler
     attr_reader :name, :bind_values_array, :statement_prepared, :prepared
 

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.1.6'
+  s.version         = '5.2.0'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -41,7 +41,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
 
   s.add_development_dependency "childprocess"
-  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-devutils', '>= 2.3'
   s.add_development_dependency 'timecop'
   s.add_development_dependency 'jdbc-derby'
 end

data/spec/inputs/integration/integ_spec.rb

@@ -66,13 +66,13 @@ describe LogStash::Inputs::Jdbc, :integration => true do
       )
     end
 
-    it "should not register correctly" do
+    it "log warning msg when plugin run" do
       plugin.register
-      allow( plugin ).to receive(:log_java_exception)
+      expect( plugin ).to receive(:log_java_exception)
+      expect(plugin.logger).to receive(:warn).once.with("Exception when executing JDBC query",
+                                                        hash_including(:exception => instance_of(String)))
       q = Queue.new
-      expect do
-        plugin.run(q)
-      end.to raise_error(::Sequel::DatabaseConnectionError)
+      expect{ plugin.run(q) }.not_to raise_error
     end
 
     it "should log (native) Java driver error" do
@@ -85,9 +85,7 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         logger
       end
       q = Queue.new
-      expect do
-        plugin.run(q)
-      end.to raise_error(::Sequel::DatabaseConnectionError)
+      expect{ plugin.run(q) }.not_to raise_error
     end
   end
 end

data/spec/inputs/jdbc_spec.rb

@@ -14,10 +14,11 @@ require "date"
 
 describe LogStash::Inputs::Jdbc do
   let(:connection_string) { "jdbc:derby:memory:testdb;create=true" }
+  let(:jdbc_driver_class) { "org.apache.derby.jdbc.EmbeddedDriver" }
   let(:mixin_settings) do
     {
         "jdbc_user" => ENV['USER'],
-        "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_driver_class" => jdbc_driver_class,
         "jdbc_connection_string" => connection_string
     }
   end
@@ -328,6 +329,39 @@ describe LogStash::Inputs::Jdbc do
 
   end
 
+  context "when iterating result-set via explicit paging mode" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from test_table OFFSET :offset ROWS FETCH NEXT :size ROWS ONLY",
+        "jdbc_paging_enabled" => true,
+        "jdbc_paging_mode" => "explicit",
+        "jdbc_page_size" => 10
+      }
+    end
+
+    let(:num_rows) { 15 }
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should fetch all rows" do
+      num_rows.times do
+        db[:test_table].insert(:num => 1, :custom_time => Time.now.utc, :created_at => Time.now.utc)
+      end
+
+      plugin.run(queue)
+
+      expect(queue.size).to eq(num_rows)
+    end
+
+  end
+
   context "when using target option" do
     let(:settings) do
       {
@@ -1392,10 +1426,8 @@ describe LogStash::Inputs::Jdbc do
       event = queue.pop
       expect(event.get("num")).to eq(1)
       expect(event.get("string")).to eq("A test")
-      expect(event.get("started_at")).to be_a(LogStash::Timestamp)
-      expect(event.get("started_at").to_s).to eq("1999-12-31T00:00:00.000Z")
-      expect(event.get("custom_time")).to be_a(LogStash::Timestamp)
-      expect(event.get("custom_time").to_s).to eq("1999-12-31T23:59:59.000Z")
+      expect(event.get("started_at")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T00:00:00.000Z")
+      expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T23:59:59.000Z")
       expect(event.get("ranking").to_f).to eq(95.67)
     end
   end
@@ -1441,10 +1473,8 @@ describe LogStash::Inputs::Jdbc do
         event = queue.pop
         expect(event.get("num")).to eq(1)
         expect(event.get("string")).to eq("A test")
-        expect(event.get("started_at")).to be_a(LogStash::Timestamp)
-        expect(event.get("started_at").to_s).to eq("1999-12-31T00:00:00.000Z")
-        expect(event.get("custom_time")).to be_a(LogStash::Timestamp)
-        expect(event.get("custom_time").to_s).to eq("1999-12-31T23:59:59.000Z")
+        expect(event.get("started_at")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T00:00:00.000Z")
+        expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T23:59:59.000Z")
         expect(event.get("ranking").to_f).to eq(95.67)
       end
     end
@@ -1619,4 +1649,40 @@ describe LogStash::Inputs::Jdbc do
       end
     end
   end
+
+  describe "jdbc_driver_class" do
+    context "when not prefixed with Java::" do
+      let(:jdbc_driver_class) { "org.apache.derby.jdbc.EmbeddedDriver" }
+      it "loads the class" do
+        expect { plugin.send(:load_driver) }.not_to raise_error
+      end
+    end
+    context "when prefixed with Java::" do
+      let(:jdbc_driver_class) { "Java::org.apache.derby.jdbc.EmbeddedDriver" }
+      it "loads the class" do
+        expect { plugin.send(:load_driver) }.not_to raise_error
+      end
+    end
+    context "when prefixed with Java." do
+      let(:jdbc_driver_class) { "Java.org::apache::derby::jdbc.EmbeddedDriver" }
+      it "loads the class" do
+        expect { plugin.send(:load_driver) }.not_to raise_error
+      end
+
+      it "can instantiate the returned driver class" do
+        # for drivers where the path through DriverManager fails, Sequel assumes
+        # having a proxied Java class instance (instead of a java.lang.Class) and
+        # does a driver.new.connect https://git.io/JDV6M
+        driver = plugin.send(:load_driver)
+        expect { driver.new }.not_to raise_error
+      end
+    end
+    context "when class name invalid" do
+      let(:jdbc_driver_class) { "org.apache.NonExistentDriver" }
+      it "raises a loading error" do
+        expect { plugin.send(:load_driver) }.to raise_error LogStash::PluginLoadingError,
+                                                            /java.lang.ClassNotFoundException: org.apache.NonExistentDriver/
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.1.6
+  version: 5.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-12 00:00:00.000000000 Z
+date: 2021-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -203,7 +203,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
   name: logstash-devutils
   prerelease: false
   type: :development
@@ -211,7 +211,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: