logstash-input-jdbc 4.3.14 → 4.3.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da0726e9534ffb2d10744efb977233487a21fb9e05e047294d5d2349e5a789fc
-  data.tar.gz: 1ea1948f5846b04ef7ecff9c9088f6d1d8c823ac38eceb07b0d83641be01666d
+  metadata.gz: 05a9c38896f4f81325e5284a4cd46085105ecc31d5339526a67a42ff668d8c41
+  data.tar.gz: b8d46982d68047d258d0587f66e53cc324c419e8a97952b75db717894e453c24
 SHA512:
-  metadata.gz: 1c5ea4dbfba07fb58625b7163740769c36b0288b8dae3a8877dc5feb2af089c71c8fc39982f17f48268bf30b290adf3f5ee942bfe7cba5bbc8f25157c538052b
-  data.tar.gz: a2d8a17d3f33a9c60fff84376566d909d2b29506a7b257c65ec739e9d1fcf953e832d039f9806698dcc34362281264c4f64b460cef41966ca95567996991cc39
+  metadata.gz: b6ab529a8eac82086d7518a1cb282189c1ad96faf138637ab168889030f09c65a6159bfc0e79eaf8c2fff8806b0e71bec46dba15ba81fe08151ac3173953202b
+  data.tar.gz: e0071180f548ceac30f79b18e28b3804c29e559c873bf4a9d45bb11b61736f7a583fbc5a9e53b11d46ed58c3e3686e50de218db89535e231ba5a631cb0b3a941

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 4.3.16
+  - Add support for prepared statements [Issue 233](https://github.com/logstash-plugins/logstash-input-jdbc/issues/233)
+
+## 4.3.15
+  - Use atomic booleam to load drivers once
+  - Added CHANGELOG entries
+
+## 4.3.14
+  - Added support for driver loading in JDK 9+ [Issue 331](https://github.com/logstash-plugins/logstash-input-jdbc/issues/331)
+  - Gem released without CHANGELOG additions
+
 ## 4.3.13
   - Changed documentation to generalize the PATH location [#297](https://github.com/logstash-plugins/logstash-input-jdbc/pull/297)
 

data/docs/index.asciidoc

@@ -140,6 +140,35 @@ input {
 }
 ---------------------------------------------------------------------------------------------------
 
+==== Prepared Statements
+
+Using server side prepared statements can speed up execution times as the server optimises the query plan and execution.
+
+NOTE: Not all JDBC accessible technologies will support prepared statements.
+
+With the introduction of Prepared Statement support comes a different code execution path and some new settings. Most of the existing settings are still useful but there are several new settings for Prepared Statements to read up on.
+Use the boolean setting `use_prepared_statements` to enable this execution mode. Use the `prepared_statement_name` setting to specify a name for the Prepared Statement, this identifies the prepared statement locally and remotely and it should be unique in your config and on the database. Use the `prepared_statement_bind_values` array setting to specify the bind values, use the exact string `:sql_last_value` (multiple times if necessary) for the predefined parameter mentioned before. The `statement` (or `statement_path`) setting still holds the SQL statement but to use bind variables you must use the `?` character as a placeholder in the exact order found in the `prepared_statement_bind_values` array.
+
+NOTE: Building count queries around a prepared statement is not supported at this time and because jdbc paging uses count queries under the hood, jdbc paging is not supported with prepared statements at this time either. Therefore, `jdbc_paging_enabled`, `jdbc_page_size` settings are ignored when using prepared statements.
+
+Example:
+[source,ruby]
+---------------------------------------------------------------------------------------------------
+input {
+  jdbc {
+    statement => "SELECT * FROM mgd.seq_sequence WHERE _sequence_key > ? AND _sequence_key < ? + ? ORDER BY _sequence_key ASC"
+    prepared_statement_bind_values => [":sql_last_value", ":sql_last_value", 4]
+    prepared_statement_name => "foobar"
+    use_prepared_statements => true
+    use_column_value => true
+    tracking_column_type => "numeric"
+    tracking_column => "_sequence_key"
+    last_run_metadata_path => "/elastic/tmp/testing/confs/test-jdbc-int-sql_last_value.yml"
+    # ... other configuration bits
+  }
+}
+---------------------------------------------------------------------------------------------------
+
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Jdbc Input Configuration Options
@@ -149,7 +178,6 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
-| <<plugins-{type}s-{plugin}-plugin_timezone>> |<<string,string>>, one of `["local", "utc"]`|No
 | <<plugins-{type}s-{plugin}-clean_run>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-columns_charset>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-connection_retry_attempts>> |<<number,number>>|No
@@ -170,6 +198,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-last_run_metadata_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-lowercase_column_names>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-plugin_timezone>> |<<string,string>>, one of `["local", "utc"]`|No
+| <<plugins-{type}s-{plugin}-prepared_statement_bind_values>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-prepared_statement_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-record_last_run>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sequel_opts>> |<<hash,hash>>|No
@@ -179,6 +210,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-tracking_column>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-tracking_column_type>> |<<string,string>>, one of `["numeric", "timestamp"]`|No
 | <<plugins-{type}s-{plugin}-use_column_value>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-use_prepared_statements>> |<<boolean,boolean>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -284,7 +316,7 @@ database timezone is UTC then you do not need to set either of these settings.
   * Value type is <<string,string>>
   * There is no default value for this setting.
 
-JDBC driver class to load, for exmaple, "org.apache.derby.jdbc.ClientDriver"
+JDBC driver class to load, for example, "org.apache.derby.jdbc.ClientDriver"
 NB per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43 if you are using
 the Oracle JDBC driver (ojdbc6.jar) the correct `jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`
 
@@ -409,6 +441,22 @@ Whether to force the lowercasing of identifier fields
 
 Hash of query parameter, for example `{ "target_id" => "321" }`
 
+[id="plugins-{type}s-{plugin}-prepared_statement_bind_values"]
+===== `prepared_statement_bind_values`
+
+  * Value type is <<array,array>>
+  * Default value is `[]`
+
+Array of bind values for the prepared statement. `:sql_last_value` is a reserved predefined string
+
+[id="plugins-{type}s-{plugin}-prepared_statement_name"]
+===== `prepared_statement_name`
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+Name given to the prepared statement. It must be unique in your config and in the database
+
 [id="plugins-{type}s-{plugin}-record_last_run"]
 ===== `record_last_run`
 
@@ -506,6 +554,13 @@ When set to `true`, uses the defined
 <<plugins-{type}s-{plugin}-tracking_column>> value as the `:sql_last_value`. When set
 to `false`, `:sql_last_value` reflects the last time the query was executed.
 
+[id="plugins-{type}s-{plugin}-use_prepared_statements"]
+===== `use_prepared_statements`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+When set to `true`, enables prepare statement usage
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/jdbc.rb

@@ -201,6 +201,12 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   # this will only convert column0 that has ISO-8859-1 as an original encoding.
   config :columns_charset, :validate => :hash, :default => {}
 
+  config :use_prepared_statements, :validate => :boolean, :default => false
+
+  config :prepared_statement_name, :validate => :string, :default => ""
+
+  config :prepared_statement_bind_values, :validate => :array, :default => []
+
   attr_reader :database # for test mocking/stubbing
 
   public
@@ -217,17 +223,25 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
       end
     end
 
-    set_value_tracker(LogStash::PluginMixins::Jdbc::ValueTracking.build_last_value_tracker(self))
-    set_statement_logger(LogStash::PluginMixins::Jdbc::CheckedCountLogger.new(@logger))
-
-    @enable_encoding = !@charset.nil? || !@columns_charset.empty?
-
     unless @statement.nil? ^ @statement_filepath.nil?
       raise(LogStash::ConfigurationError, "Must set either :statement or :statement_filepath. Only one may be set at a time.")
     end
 
     @statement = ::File.read(@statement_filepath) if @statement_filepath
 
+    # must validate prepared statement mode after trying to read in from @statement_filepath
+    if @use_prepared_statements
+      validation_errors = validate_prepared_statement_mode
+      unless validation_errors.empty?
+        raise(LogStash::ConfigurationError, "Prepared Statement Mode validation errors: " + validation_errors.join(", "))
+      end
+    end
+
+    set_value_tracker(LogStash::PluginMixins::Jdbc::ValueTracking.build_last_value_tracker(self))
+    set_statement_logger(LogStash::PluginMixins::Jdbc::CheckedCountLogger.new(@logger))
+
+    @enable_encoding = !@charset.nil? || !@columns_charset.empty?
+
     if (@jdbc_password_filepath and @jdbc_password)
       raise(LogStash::ConfigurationError, "Only one of :jdbc_password, :jdbc_password_filepath may be set at a time.")
     end
@@ -248,7 +262,7 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
 
   # test injection points
   def set_statement_logger(instance)
-    @statement_logger = instance
+    @statement_handler = LogStash::PluginMixins::Jdbc::StatementHandler.build_statement_handler(self, instance)
   end
 
   def set_value_tracker(instance)
@@ -275,10 +289,24 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
 
   private
 
+  def validate_prepared_statement_mode
+    error_messages = []
+    if @prepared_statement_name.empty?
+      error_messages << "must provide a name for the Prepared Statement, it must be unique for the db session"
+    end
+    if @statement.count("?") != @prepared_statement_bind_values.size
+      # mismatch in number of bind value elements to placeholder characters
+      error_messages << "there is a mismatch between the number of statement `?` placeholders and :prepared_statement_bind_values array setting elements"
+    end
+    if @jdbc_paging_enabled
+      # Pagination is not supported when using prepared statements
+      error_messages << "JDBC pagination cannot be used at this time"
+    end
+    error_messages
+  end
+
   def execute_query(queue)
-    # update default parameters
-    @parameters['sql_last_value'] = @value_tracker.value
-    execute_statement(@statement, @parameters) do |row|
+    execute_statement do |row|
       if enable_encoding?
         ## do the necessary conversions to string elements
         row = Hash[row.map { |k, v| [k.to_s, convert(k, v)] }]

data/lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb

@@ -9,9 +9,14 @@ module LogStash module PluginMixins module Jdbc
       @in_debug = @logger.debug?
     end
 
-    def log_statement_parameters(query, statement, parameters)
+    def disable_count
+      @needs_check = false
+      @count_is_supported = false
+    end
+
+    def log_statement_parameters(statement, parameters, query)
       return unless @in_debug
-      check_count_query(query) if @needs_check
+      check_count_query(query) if @needs_check && query
       if @count_is_supported
         @logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters, :count => execute_count(query))
       else
@@ -35,4 +40,4 @@ module LogStash module PluginMixins module Jdbc
       query.count
     end
   end
-end end end
+end end end

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -6,6 +6,7 @@ require "date"
 require_relative "value_tracking"
 require_relative "checked_count_logger"
 require_relative "wrapped_driver"
+require_relative "statement_handler"
 
 java_import java.util.concurrent.locks.ReentrantLock
 
@@ -171,22 +172,24 @@ module LogStash  module PluginMixins module Jdbc
       require "sequel/adapters/jdbc"
 
       Sequel.application_timezone = @plugin_timezone.to_sym
-
-      begin
-        load_drivers
-        Sequel::JDBC.load_driver(@jdbc_driver_class)
-      rescue LogStash::Error => e
-        # raised in load_drivers, e.cause should be the caught Java exceptions
-        raise LogStash::PluginLoadingError, "#{e.message} and #{e.cause.message}"
-      rescue Sequel::AdapterNotFound => e
-        # fix this !!!
-        message = if @jdbc_driver_library.nil?
-          ":jdbc_driver_library is not set, are you sure you included
-                    the proper driver client libraries in your classpath?"
-        else
-          "Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
+      if @drivers_loaded.false?
+        begin
+          load_drivers
+          Sequel::JDBC.load_driver(@jdbc_driver_class)
+        rescue LogStash::Error => e
+          # raised in load_drivers, e.cause should be the caught Java exceptions
+          raise LogStash::PluginLoadingError, "#{e.message} and #{e.cause.message}"
+        rescue Sequel::AdapterNotFound => e
+          # fix this !!!
+          message = if @jdbc_driver_library.nil?
+            ":jdbc_driver_library is not set, are you sure you included
+                      the proper driver client libraries in your classpath?"
+          else
+            "Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
+          end
+          raise LogStash::PluginLoadingError, "#{e}. #{message}"
         end
-        raise LogStash::PluginLoadingError, "#{e}. #{message}"
+        @drivers_loaded.make_true
       end
       @database = jdbc_connect()
       @database.extension(:pagination)
@@ -226,6 +229,7 @@ module LogStash  module PluginMixins module Jdbc
     public
     def prepare_jdbc_connection
       @connection_lock = ReentrantLock.new
+      @drivers_loaded = Concurrent::AtomicBoolean.new
     end
 
     public
@@ -243,18 +247,14 @@ module LogStash  module PluginMixins module Jdbc
     end
 
     public
-    def execute_statement(statement, parameters)
-      # sql_last_value has been set in params by caller
+    def execute_statement
       success = false
       @connection_lock.lock
       open_jdbc_connection
       begin
-        params = symbolized_params(parameters)
-        query = @database[statement, params]
         sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
         @tracking_column_warning_sent = false
-        @statement_logger.log_statement_parameters(query, statement, params)
-        perform_query(query) do |row|
+        @statement_handler.perform_query(@database, @value_tracker.value) do |row|
           sql_last_value = get_column_value(row) if @use_column_value
           yield extract_values_from(row)
         end
@@ -270,24 +270,6 @@ module LogStash  module PluginMixins module Jdbc
       return success
     end
 
-# Performs the query, respecting our pagination settings, yielding once per row of data
-# @param query [Sequel::Dataset]
-# @yieldparam row [Hash{Symbol=>Object}]
-    private
-    def perform_query(query)
-      if @jdbc_paging_enabled
-        query.each_page(@jdbc_page_size) do |paged_dataset|
-          paged_dataset.each do |row|
-            yield row
-          end
-        end
-      else
-        query.each do |row|
-          yield row
-        end
-      end
-    end
-
     public
     def get_column_value(row)
       if !row.has_key?(@tracking_column.to_sym)
@@ -295,7 +277,7 @@ module LogStash  module PluginMixins module Jdbc
           @logger.warn("tracking_column not found in dataset.", :tracking_column => @tracking_column)
           @tracking_column_warning_sent = true
         end
-        # If we can't find the tracking column, return the current value_tracker value
+        # If we can't find the tracking column, return the current value in the ivar
         @value_tracker.value
       else
         # Otherwise send the updated tracking column
@@ -303,22 +285,8 @@ module LogStash  module PluginMixins module Jdbc
       end
     end
 
-# Symbolize parameters keys to use with Sequel
-    private
-    def symbolized_params(parameters)
-      parameters.inject({}) do |hash,(k,v)|
-        case v
-        when LogStash::Timestamp
-          hash[k.to_sym] = v.time
-        else
-          hash[k.to_sym] = v
-        end
-        hash
-      end
-    end
-
     private
-#Stringify row keys and decorate values when necessary
+    #Stringify row keys and decorate values when necessary
     def extract_values_from(row)
       Hash[row.map { |k, v| [k.to_s, decorate_value(v)] }]
     end

data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+
+module LogStash module PluginMixins module Jdbc
+  class StatementHandler
+    def self.build_statement_handler(plugin, logger)
+      klass = plugin.use_prepared_statements ? PreparedStatementHandler : NormalStatementHandler
+      klass.new(plugin, logger)
+    end
+
+    attr_reader :statement, :parameters, :statement_logger
+
+    def initialize(plugin, statement_logger)
+      @statement = plugin.statement
+      @statement_logger = statement_logger
+      post_init(plugin)
+    end
+
+    def build_query(db, sql_last_value)
+      # override in subclass
+    end
+
+    def post_init(plugin)
+      # override in subclass, if needed
+    end
+  end
+
+  class NormalStatementHandler < StatementHandler
+    # Performs the query, respecting our pagination settings, yielding once per row of data
+    # @param db [Sequel::Database]
+    # @param sql_last_value [Integet|DateTime|Time]
+    # @yieldparam row [Hash{Symbol=>Object}]
+    def perform_query(db, sql_last_value)
+      query = build_query(db, sql_last_value)
+      if @jdbc_paging_enabled
+        query.each_page(@jdbc_page_size) do |paged_dataset|
+          paged_dataset.each do |row|
+            yield row
+          end
+        end
+      else
+        query.each do |row|
+          yield row
+        end
+      end
+    end
+
+    private
+
+    def build_query(db, sql_last_value)
+      parameters[:sql_last_value] = sql_last_value
+      query = db[statement, parameters]
+      statement_logger.log_statement_parameters(statement, parameters, query)
+      query
+    end
+
+    def post_init(plugin)
+      @parameter_keys = ["sql_last_value"] + plugin.parameters.keys
+      @parameters = plugin.parameters.inject({}) do |hash,(k,v)|
+        case v
+        when LogStash::Timestamp
+          hash[k.to_sym] = v.time
+        else
+          hash[k.to_sym] = v
+        end
+        hash
+      end
+    end
+  end
+
+  class PreparedStatementHandler < StatementHandler
+    attr_reader :name, :bind_values_array, :statement_prepared, :prepared
+
+    # Performs the query, ignoring our pagination settings, yielding once per row of data
+    # @param db [Sequel::Database]
+    # @param sql_last_value [Integet|DateTime|Time]
+    # @yieldparam row [Hash{Symbol=>Object}]
+    def perform_query(db, sql_last_value)
+      query = build_query(db, sql_last_value)
+      query.each do |row|
+        yield row
+      end
+    end
+
+    private
+
+    def build_query(db, sql_last_value)
+      @parameters = create_bind_values_hash
+      if statement_prepared.false?
+        prepended = parameters.keys.map{|v| v.to_s.prepend("$").to_sym}
+        @prepared = db[statement, *prepended].prepare(:select, name)
+        statement_prepared.make_true
+      end
+      # under the scheduler the Sequel database instance is recreated each time
+      # so the previous prepared statements are lost, add back
+      if db.prepared_statement(name).nil?
+        db.set_prepared_statement(name, prepared)
+      end
+      bind_value_sql_last_value(sql_last_value)
+      statement_logger.log_statement_parameters(statement, parameters, nil)
+      db.call(name, parameters)
+    end
+
+    def post_init(plugin)
+      # don't log statement count when using prepared statements for now...
+      # needs enhancement to allow user to supply a bindable count prepared statement in settings.
+      @statement_logger.disable_count
+
+      @name = plugin.prepared_statement_name.to_sym
+      @bind_values_array = plugin.prepared_statement_bind_values
+      @parameters = plugin.parameters
+      @statement_prepared = Concurrent::AtomicBoolean.new(false)
+    end
+
+    def create_bind_values_hash
+      hash = {}
+      bind_values_array.each_with_index {|v,i| hash[:"p#{i}"] = v}
+      hash
+    end
+
+    def bind_value_sql_last_value(sql_last_value)
+      parameters.keys.each do |key|
+        value = parameters[key]
+        if value == ":sql_last_value"
+          parameters[key] = sql_last_value
+        end
+      end
+    end
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -5,28 +5,26 @@ module LogStash module PluginMixins module Jdbc
   class ValueTracking
 
     def self.build_last_value_tracker(plugin)
+      handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_path) : NullFileHandler.new(plugin.last_run_metadata_path)
+      if plugin.record_last_run
+        handler = FileHandler.new(plugin.last_run_metadata_path)
+      end
+      if plugin.clean_run
+        handler.clean
+      end
+
       if plugin.use_column_value && plugin.tracking_column_type == "numeric"
         # use this irrespective of the jdbc_default_timezone setting
-        klass = NumericValueTracker
+        NumericValueTracker.new(handler)
       else
         if plugin.jdbc_default_timezone.nil? || plugin.jdbc_default_timezone.empty?
           # no TZ stuff for Sequel, use Time
-          klass = TimeValueTracker
+          TimeValueTracker.new(handler)
         else
           # Sequel does timezone handling on DateTime only
-          klass = DateTimeValueTracker
+          DateTimeValueTracker.new(handler)
         end
       end
-
-      handler = NullFileHandler.new(plugin.last_run_metadata_path)
-      if plugin.record_last_run
-        handler = FileHandler.new(plugin.last_run_metadata_path)
-      end
-      if plugin.clean_run
-        handler.clean
-      end
-
-      instance = klass.new(handler)
     end
 
     attr_reader :value
@@ -51,6 +49,7 @@ module LogStash module PluginMixins module Jdbc
     private
     def common_set_initial(method_symbol, default)
       persisted = @file_handler.read
+
       if persisted && persisted.respond_to?(method_symbol)
         @value = persisted
       else

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '4.3.14'
+  s.version         = '4.3.16'
   s.licenses = ['Apache License (2.0)']
   s.summary = "Creates events from JDBC data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -1310,4 +1310,142 @@ describe LogStash::Inputs::Jdbc do
       end
     end
   end
+
+  context "when using prepared statements" do
+    let(:last_run_value) { 250 }
+    let(:expected_queue_size) { 100 }
+    let(:num_rows) { 1000 }
+
+    context "check validation" do
+      context "with an empty name setting" do
+        let(:settings) do
+          {
+            "statement" => "SELECT * FROM test_table ORDER BY num FETCH NEXT ? ROWS ONLY",
+            "prepared_statement_bind_values" => [100],
+            "use_prepared_statements" => true,
+          }
+        end
+
+        it "should fail to register" do
+          expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
+        end
+      end
+
+      context "with an mismatched placeholder vs bind values" do
+        let(:settings) do
+          {
+            "statement" => "SELECT * FROM test_table ORDER BY num FETCH NEXT ? ROWS ONLY",
+            "prepared_statement_bind_values" => [],
+            "use_prepared_statements" => true,
+          }
+        end
+
+        it "should fail to register" do
+          expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
+        end
+      end
+
+      context "with jdbc paging enabled" do
+        let(:settings) do
+          {
+            "statement" => "SELECT * FROM test_table ORDER BY num FETCH NEXT 100 ROWS ONLY",
+            "prepared_statement_bind_values" => [],
+            "prepared_statement_name" => "pstmt_test_without",
+            "use_prepared_statements" => true,
+            "jdbc_paging_enabled" => true,
+            "jdbc_page_size" => 20,
+            "jdbc_fetch_size" => 10
+          }
+        end
+
+        it "should fail to register" do
+          expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
+        end
+      end
+
+    end
+
+    context "and no validation failures" do
+      before do
+        ::File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+        num_rows.times do |n|
+          db[:test_table].insert(:num => n.succ, :string => SecureRandom.hex(8), :custom_time => Time.now.utc, :created_at => Time.now.utc)
+        end
+      end
+
+      after do
+        plugin.stop
+      end
+
+      context "with jdbc paging enabled" do
+        let(:settings) do
+          {
+            "statement" => "SELECT * FROM test_table ORDER BY num FETCH NEXT 100 ROWS ONLY",
+            "prepared_statement_bind_values" => [],
+            "prepared_statement_name" => "pstmt_test_without",
+            "use_prepared_statements" => true,
+            "tracking_column_type" => "numeric",
+            "tracking_column" => "num",
+            "use_column_value" => true,
+            "last_run_metadata_path" => Stud::Temporary.pathname,
+            "jdbc_paging_enabled" => true,
+            "jdbc_page_size" => 20,
+            "jdbc_fetch_size" => 10
+          }
+        end
+
+        it "should fail to register" do
+          expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
+        end
+      end
+
+      context "without placeholder and bind parameters" do
+        let(:settings) do
+          {
+            "statement" => "SELECT * FROM test_table ORDER BY num FETCH NEXT 100 ROWS ONLY",
+            "prepared_statement_bind_values" => [],
+            "prepared_statement_name" => "pstmt_test_without",
+            "use_prepared_statements" => true,
+            "tracking_column_type" => "numeric",
+            "tracking_column" => "num",
+            "use_column_value" => true,
+            "last_run_metadata_path" => Stud::Temporary.pathname
+          }
+        end
+
+        it "should fetch some rows" do
+          plugin.register
+          plugin.run(queue)
+
+          expect(queue.size).to eq(expected_queue_size)
+          expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
+        end
+      end
+
+
+      context "with bind parameters" do
+        let(:settings) do
+          {
+            # Sadly, postgres does but derby doesn't - It is not allowed for both operands of '+' to be ? parameters.: PREPARE pstmt_test: SELECT * FROM test_table WHERE (num > ?) AND (num <= ? + ?) ORDER BY num
+            "statement" => "SELECT * FROM test_table WHERE (num > ?) ORDER BY num FETCH NEXT ? ROWS ONLY",
+            "prepared_statement_bind_values" => [":sql_last_value", expected_queue_size],
+            "prepared_statement_name" => "pstmt_test_with",
+            "use_prepared_statements" => true,
+            "tracking_column_type" => "numeric",
+            "tracking_column" => "num",
+            "use_column_value" => true,
+            "last_run_metadata_path" => Stud::Temporary.pathname
+          }
+        end
+
+        it "should fetch some rows" do
+          plugin.register
+          plugin.run(queue)
+
+          expect(queue.size).to eq(expected_queue_size)
+          expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
+        end
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 4.3.14
+  version: 4.3.16
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-08-29 00:00:00.000000000 Z
+date: 2019-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -174,6 +174,7 @@ files:
 - lib/logstash/inputs/tzinfo_jruby_patch.rb
 - lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb
 - lib/logstash/plugin_mixins/jdbc/jdbc.rb
+- lib/logstash/plugin_mixins/jdbc/statement_handler.rb
 - lib/logstash/plugin_mixins/jdbc/value_tracking.rb
 - lib/logstash/plugin_mixins/jdbc/wrapped_driver.rb
 - logstash-input-jdbc.gemspec