logstash-integration-jdbc 5.2.4 → 5.4.11

data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb

@@ -2,7 +2,7 @@
 
 module LogStash module PluginMixins module Jdbc
   class StatementHandler
-    def self.build_statement_handler(plugin, logger)
+    def self.build_statement_handler(plugin)
       if plugin.use_prepared_statements
         klass = PreparedStatementHandler
       else
@@ -16,27 +16,39 @@ module LogStash module PluginMixins module Jdbc
           klass = NormalStatementHandler
         end
       end
-      klass.new(plugin, logger)
+      klass.new(plugin)
     end
 
-    attr_reader :statement, :parameters, :statement_logger
+    attr_reader :statement, :parameters
 
-    def initialize(plugin, statement_logger)
+    def initialize(plugin)
       @statement = plugin.statement
-      @statement_logger = statement_logger
-      post_init(plugin)
     end
 
     def build_query(db, sql_last_value)
-      # override in subclass
+      fail NotImplementedError # override in subclass
     end
 
-    def post_init(plugin)
-      # override in subclass, if needed
-    end
   end
 
   class NormalStatementHandler < StatementHandler
+
+    attr_reader :parameters
+
+    def initialize(plugin)
+      super(plugin)
+      @parameter_keys = ["sql_last_value"] + plugin.parameters.keys
+      @parameters = plugin.parameters.inject({}) do |hash,(k,v)|
+        case v
+        when LogStash::Timestamp
+          hash[k.to_sym] = v.time
+        else
+          hash[k.to_sym] = v
+        end
+        hash
+      end
+    end
+
     # Performs the query, yielding once per row of data
     # @param db [Sequel::Database]
     # @param sql_last_value [Integer|DateTime|Time]
@@ -52,27 +64,18 @@ module LogStash module PluginMixins module Jdbc
 
     def build_query(db, sql_last_value)
       parameters[:sql_last_value] = sql_last_value
-      query = db[statement, parameters]
-      statement_logger.log_statement_parameters(statement, parameters, query)
-      query
+      db[statement, parameters]
     end
 
-    def post_init(plugin)
-      @parameter_keys = ["sql_last_value"] + plugin.parameters.keys
-      @parameters = plugin.parameters.inject({}) do |hash,(k,v)|
-        case v
-        when LogStash::Timestamp
-          hash[k.to_sym] = v.time
-        else
-          hash[k.to_sym] = v
-        end
-        hash
-      end
-    end
   end
 
   class PagedNormalStatementHandler < NormalStatementHandler
-    attr_reader :jdbc_page_size
+
+    def initialize(plugin)
+      super(plugin)
+      @jdbc_page_size = plugin.jdbc_page_size
+      @logger = plugin.logger
+    end
 
     # Performs the query, respecting our pagination settings, yielding once per row of data
     # @param db [Sequel::Database]
@@ -81,16 +84,22 @@ module LogStash module PluginMixins module Jdbc
     def perform_query(db, sql_last_value)
       query = build_query(db, sql_last_value)
       query.each_page(@jdbc_page_size) do |paged_dataset|
+        log_dataset_page(paged_dataset) if @logger.debug?
         paged_dataset.each do |row|
           yield row
         end
       end
     end
 
-    def post_init(plugin)
-      super(plugin)
-      @jdbc_page_size = plugin.jdbc_page_size
+    private
+
+    # @param paged_dataset [Sequel::Dataset::Pagination] like object
+    def log_dataset_page(paged_dataset)
+      @logger.debug "fetching paged dataset", current_page: paged_dataset.current_page,
+                                              record_count: paged_dataset.current_page_record_count,
+                                              total_record_count: paged_dataset.pagination_record_count
     end
+
   end
 
   class ExplicitPagingModeStatementHandler < PagedNormalStatementHandler
@@ -101,20 +110,29 @@ module LogStash module PluginMixins module Jdbc
     def perform_query(db, sql_last_value)
       query = build_query(db, sql_last_value)
       offset = 0
+      page_size = @jdbc_page_size
       loop do
         rows_in_page = 0
-        query.with_sql(query.sql, offset: offset, size: jdbc_page_size).each do |row|
+        query.with_sql(query.sql, offset: offset, size: page_size).each do |row|
           yield row
           rows_in_page += 1
         end
-        break unless rows_in_page == jdbc_page_size
-        offset += jdbc_page_size
+        break unless rows_in_page == page_size
+        offset += page_size
       end
     end
   end
 
   class PreparedStatementHandler < StatementHandler
-    attr_reader :name, :bind_values_array, :statement_prepared, :prepared
+    attr_reader :name, :bind_values_array, :statement_prepared, :prepared, :parameters
+
+    def initialize(plugin)
+      super(plugin)
+      @name = plugin.prepared_statement_name.to_sym
+      @bind_values_array = plugin.prepared_statement_bind_values
+      @parameters = plugin.parameters
+      @statement_prepared = Concurrent::AtomicBoolean.new(false)
+    end
 
     # Performs the query, ignoring our pagination settings, yielding once per row of data
     # @param db [Sequel::Database]
@@ -142,7 +160,6 @@ module LogStash module PluginMixins module Jdbc
         db.set_prepared_statement(name, prepared)
       end
       bind_value_sql_last_value(sql_last_value)
-      statement_logger.log_statement_parameters(statement, parameters, nil)
       begin
         db.call(name, parameters)
       rescue => e
@@ -153,17 +170,6 @@ module LogStash module PluginMixins module Jdbc
       end
     end
 
-    def post_init(plugin)
-      # don't log statement count when using prepared statements for now...
-      # needs enhancement to allow user to supply a bindable count prepared statement in settings.
-      @statement_logger.disable_count
-
-      @name = plugin.prepared_statement_name.to_sym
-      @bind_values_array = plugin.prepared_statement_bind_values
-      @parameters = plugin.parameters
-      @statement_prepared = Concurrent::AtomicBoolean.new(false)
-    end
-
     def create_bind_values_hash
       hash = {}
       bind_values_array.each_with_index {|v,i| hash[:"p#{i}"] = v}

data/lib/logstash/plugin_mixins/jdbc/timezone_proxy.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+
+require 'tzinfo'
+
+module LogStash module PluginMixins module Jdbc
+  ##
+  # This `TimezoneProxy` allows timezone specs to include extensions indicating preference for ambiguous handling.
+  # @see TimezoneProxy::parse
+  module TimezoneProxy
+    ##
+    # @param timezone_spec [String]: a timezone spec, consisting of any valid timezone identifier
+    #                                followed by square-bracketed extensions. Currently-supported
+    #                                extensions are:
+    #                                `dst_enabled_on_overlap:(true|false)`: when encountering an ambiguous time
+    #                                                                       due to daylight-savings transition,
+    #                                                                       assume DST to be either enabled or
+    #                                                                       disabled instead of raising an
+    #                                                                       AmbiguousTime exception
+    # @return [TZInfo::Timezone]
+    def self.load(timezone_spec)
+      # re-load pass-through
+      return timezone_spec if timezone_spec.kind_of?(::TZInfo::Timezone)
+
+      parsed_spec = /\A(?<name>[^\[]+)(\[(?<extensions>[^\]]*)\])?\z/.match(timezone_spec)
+
+      timezone = ::TZInfo::Timezone.get(parsed_spec[:name])
+      return timezone unless parsed_spec[:extensions]
+
+      parsed_spec[:extensions].split(';').each do |extension_spec|
+        timezone = case extension_spec
+                   when 'dst_enabled_on_overlap:true'  then timezone.dup.extend(PeriodForLocalWithDSTPreference::ON)
+                   when 'dst_enabled_on_overlap:false' then timezone.dup.extend(PeriodForLocalWithDSTPreference::OFF)
+                   else fail(ArgumentError, "Invalid timezone extension `#{extension_spec}`")
+                   end
+      end
+
+      timezone
+    end
+
+    module JDBCTimezoneSpecValidator
+      def validate_value(value, validator_name)
+        return super(value, validator_name) unless validator_name == :jdbc_timezone_spec
+
+        [true, TimezoneProxy.load(value)] rescue [false, $!.message]
+      end
+    end
+
+    ##
+    # @api private
+    class PeriodForLocalWithDSTPreference < Module
+      def initialize(default_dst_enabled_on_overlap)
+        define_method(:period_for_local) do |localtime, dst_enabled_on_overlap=nil, &dismabiguation_block|
+          super(localtime, dst_enabled_on_overlap.nil? ? default_dst_enabled_on_overlap : dst_enabled_on_overlap, &dismabiguation_block)
+        end
+      end
+
+      ON = new(true)
+      OFF = new(false)
+    end
+  end
+end; end; end

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -1,11 +1,13 @@
 # encoding: utf-8
 require "yaml" # persistence
+require "date"
+require "bigdecimal"
 
 module LogStash module PluginMixins module Jdbc
   class ValueTracking
 
     def self.build_last_value_tracker(plugin)
-      handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_path) : NullFileHandler.new(plugin.last_run_metadata_path)
+      handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_file_path) : NullFileHandler.new(plugin.last_run_metadata_file_path)
       if plugin.clean_run
         handler.clean
       end
@@ -14,7 +16,7 @@ module LogStash module PluginMixins module Jdbc
         # use this irrespective of the jdbc_default_timezone setting
         NumericValueTracker.new(handler)
       else
-        if plugin.jdbc_default_timezone.nil? || plugin.jdbc_default_timezone.empty?
+        if plugin.jdbc_default_timezone.nil?
           # no TZ stuff for Sequel, use Time
           TimeValueTracker.new(handler)
         else
@@ -31,6 +33,17 @@ module LogStash module PluginMixins module Jdbc
       set_initial
     end
 
+    if Psych::VERSION&.split('.')&.first.to_i >= 4
+      YAML_PERMITTED_CLASSES = [::DateTime, ::Time, ::BigDecimal].freeze
+      def self.load_yaml(source)
+        Psych::safe_load(source, permitted_classes: YAML_PERMITTED_CLASSES)
+      end
+    else
+      def self.load_yaml(source)
+        YAML::load(source)
+      end
+    end
+
     def set_initial
       # override in subclass
     end
@@ -112,7 +125,7 @@ module LogStash module PluginMixins module Jdbc
 
     def read
       return unless @exists
-      YAML.load(::File.read(@path))
+      ValueTracking.load_yaml(::File.read(@path))
     end
 
     def write(value)

data/lib/logstash-integration-jdbc_jars.rb

@@ -1,5 +1,7 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('org.apache.derby', 'derby', '10.14.1.0')
-require_jar('org.apache.derby', 'derbyclient', '10.14.1.0')
+require_jar('org.apache.derby', 'derby', '10.15.2.1')
+require_jar('org.apache.derby', 'derbyclient', '10.15.2.1')
+require_jar('org.apache.derby', 'derbytools', '10.15.2.1')
+require_jar('org.apache.derby', 'derbyshared', '10.15.2.1')

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.2.4'
+  s.version         = '5.4.11'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -29,20 +29,20 @@ Gem::Specification.new do |s|
   # Restrict use of this plugin to versions of Logstash where support for integration plugins is present.
   s.add_runtime_dependency "logstash-core", ">= 6.5.0"
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'sequel'
+  # Pinned sequel version to >= 5.74.0 as it fixes the generic jdbc adapter to properly
+  # handle disconnection errors, and avoid stale connections in the pool.
+  s.add_runtime_dependency 'sequel', '>= 5.74.0'
   s.add_runtime_dependency 'lru_redux' # lru cache with ttl
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
-  # plugin maintains compatibility with < 3.5 (3.0.9)
-  # but works with newer rufus-scheduler >= 3.5 as well
-  s.add_runtime_dependency 'rufus-scheduler'
+
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.3'
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
+  s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
 
   s.add_development_dependency "childprocess"
   s.add_development_dependency 'logstash-devutils', '>= 2.3'
   s.add_development_dependency 'timecop'
-  s.add_development_dependency 'jdbc-derby'
 end

data/spec/filters/jdbc_static_spec.rb

@@ -5,6 +5,7 @@ require "sequel"
 require "sequel/adapters/jdbc"
 require "stud/temporary"
 require "timecop"
+require "pathname"
 
 # LogStash::Logging::Logger::configure_logging("WARN")
 
@@ -85,6 +86,15 @@ module LogStash module Filters
 
     let(:ipaddr) { ".3.1.1" }
 
+    describe "verify derby path property" do
+      it "should be set into Logstash data path" do
+        plugin.register
+
+        expected = Pathname.new(LogStash::SETTINGS.get_value("path.data")).join("plugins", "shared", "derby_home").to_path
+        expect(java.lang.System.getProperty("derby.system.home")).to eq(expected)
+      end
+    end
+
     describe "non scheduled operation" do
       after { plugin.close }
 

data/spec/filters/jdbc_streaming_spec.rb

@@ -1,7 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/devutils/rspec/shared_examples"
 require "logstash/filters/jdbc_streaming"
-require 'jdbc/derby'
 require "sequel"
 require "sequel/adapters/jdbc"
 
@@ -13,8 +12,6 @@ module LogStash module Filters
   describe JdbcStreaming do
     let!(:jdbc_connection_string) { "jdbc:derby:memory:jdbc_streaming_testdb;create=true"}
     #Use embedded Derby for tests
-    ::Jdbc::Derby.load_driver
-
     ENV["TZ"] = "Etc/UTC"
     describe "plugin level execution" do
       let(:mixin_settings) do
@@ -259,7 +256,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
     end
@@ -277,7 +274,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"col_1" => 'from_database'}])
       end
     end
@@ -296,11 +293,11 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text", "param_field" => "1") do
+      sample({"message" => "some text", "param_field" => "1"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
 
-      sample("message" => "some text", "param_field" => "2") do
+      sample({"message" => "some text", "param_field" => "2"}) do
         expect(subject.get('new_field').nil?)
       end
     end
@@ -319,11 +316,11 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text", "param_field" => 1) do
+      sample({"message" => "some text", "param_field" => 1}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
 
-      sample("message" => "some text", "param_field" => "1") do
+      sample({"message" => "some text", "param_field" => "1"}) do
         expect(subject.get('new_field').nil?)
       end
     end
@@ -342,7 +339,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
     end

data/spec/inputs/integration/integ_spec.rb

@@ -41,6 +41,31 @@ describe LogStash::Inputs::Jdbc, :integration => true do
       expect(event.get('first_name')).to eq("Mark")
       expect(event.get('last_name')).to eq("Guckenheimer")
     end
+
+    context 'with paging' do
+      let(:settings) do
+        super().merge 'jdbc_paging_enabled' => true, 'jdbc_page_size' => 1,
+                      "statement" => 'SELECT * FROM "employee" WHERE EMP_NO >= :p1 ORDER BY EMP_NO',
+                      'parameters' => { 'p1' => 0 }
+      end
+
+      before do # change plugin logger level to debug - to exercise logging
+        logger = plugin.class.name.gsub('::', '.').downcase
+        logger = org.apache.logging.log4j.LogManager.getLogger(logger)
+        @prev_logger_level = [ logger.getName, logger.getLevel ]
+        org.apache.logging.log4j.core.config.Configurator.setLevel logger.getName, org.apache.logging.log4j.Level::DEBUG
+      end
+
+      after do
+        org.apache.logging.log4j.core.config.Configurator.setLevel *@prev_logger_level
+      end
+
+      it "should populate the event with database entries" do
+        plugin.run(queue)
+        event = queue.pop
+        expect(event.get('first_name')).to eq('David')
+      end
+    end
   end
 
   context "when supplying a non-existent library" do
@@ -51,10 +76,8 @@ describe LogStash::Inputs::Jdbc, :integration => true do
     end
 
     it "should not register correctly" do
-      plugin.register
-      q = Queue.new
       expect do
-        plugin.run(q)
+        plugin.register
       end.to raise_error(::LogStash::PluginLoadingError)
     end
   end
@@ -67,16 +90,13 @@ describe LogStash::Inputs::Jdbc, :integration => true do
     end
 
     it "log warning msg when plugin run" do
-      plugin.register
       expect( plugin ).to receive(:log_java_exception)
       expect(plugin.logger).to receive(:warn).once.with("Exception when executing JDBC query",
                                                         hash_including(:message => instance_of(String)))
-      q = Queue.new
-      expect{ plugin.run(q) }.not_to raise_error
+      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
     end
 
     it "should log (native) Java driver error" do
-      plugin.register
       expect( org.apache.logging.log4j.LogManager ).to receive(:getLogger).and_wrap_original do |m, *args|
         logger = m.call(*args)
         expect( logger ).to receive(:error) do |_, e|
@@ -84,8 +104,7 @@ describe LogStash::Inputs::Jdbc, :integration => true do
         end.and_call_original
         logger
       end
-      q = Queue.new
-      expect{ plugin.run(q) }.not_to raise_error
+      expect{ plugin.register }.to raise_error(::LogStash::ConfigurationError)
     end
   end
 end