logstash-integration-jdbc 5.0.0.alpha1

data/lib/logstash/plugin_mixins/jdbc/jdbc.rb

@@ -0,0 +1,298 @@
+# encoding: utf-8
+# TAKEN FROM WIIBAA
+require "logstash/config/mixin"
+require "time"
+require "date"
+require_relative "value_tracking"
+require_relative "checked_count_logger"
+require_relative "statement_handler"
+
+java_import java.util.concurrent.locks.ReentrantLock
+
+# Tentative of abstracting JDBC logic to a mixin
+# for potential reuse in other plugins (input/output)
+module LogStash  module PluginMixins module Jdbc
+  module Jdbc
+    # This method is called when someone includes this module
+    def self.included(base)
+      # Add these methods to the 'base' given.
+      base.extend(self)
+      base.setup_jdbc_config
+    end
+
+
+    public
+    def setup_jdbc_config
+      # JDBC driver library path to third party driver library. In case of multiple libraries being
+      # required you can pass them separated by a comma.
+      #
+      # If not provided, Plugin will look for the driver class in the Logstash Java classpath.
+      config :jdbc_driver_library, :validate => :string
+
+      # JDBC driver class to load, for exmaple, "org.apache.derby.jdbc.ClientDriver"
+      # NB per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43 if you are using
+      # the Oracle JDBC driver (ojdbc6.jar) the correct `jdbc_driver_class` is `"Java::oracle.jdbc.driver.OracleDriver"`
+      config :jdbc_driver_class, :validate => :string, :required => true
+
+      # JDBC connection string
+      config :jdbc_connection_string, :validate => :string, :required => true
+
+      # JDBC user
+      config :jdbc_user, :validate => :string, :required => true
+
+      # JDBC password
+      config :jdbc_password, :validate => :password
+
+      # JDBC password filename
+      config :jdbc_password_filepath, :validate => :path
+
+      # JDBC enable paging
+      #
+      # This will cause a sql statement to be broken up into multiple queries.
+      # Each query will use limits and offsets to collectively retrieve the full
+      # result-set. The limit size is set with `jdbc_page_size`.
+      #
+      # Be aware that ordering is not guaranteed between queries.
+      config :jdbc_paging_enabled, :validate => :boolean, :default => false
+
+      # JDBC page size
+      config :jdbc_page_size, :validate => :number, :default => 100000
+
+      # JDBC fetch size. if not provided, respective driver's default will be used
+      config :jdbc_fetch_size, :validate => :number
+
+      # Connection pool configuration.
+      # Validate connection before use.
+      config :jdbc_validate_connection, :validate => :boolean, :default => false
+
+      # Connection pool configuration.
+      # How often to validate a connection (in seconds)
+      config :jdbc_validation_timeout, :validate => :number, :default => 3600
+
+      # Connection pool configuration.
+      # The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5)
+      config :jdbc_pool_timeout, :validate => :number, :default => 5
+
+      # Timezone conversion.
+      # SQL does not allow for timezone data in timestamp fields.  This plugin will automatically
+      # convert your SQL timestamp fields to Logstash timestamps, in relative UTC time in ISO8601 format.
+      #
+      # Using this setting will manually assign a specified timezone offset, instead
+      # of using the timezone setting of the local machine.  You must use a canonical
+      # timezone, *America/Denver*, for example.
+      config :jdbc_default_timezone, :validate => :string
+
+      # General/Vendor-specific Sequel configuration options.
+      #
+      # An example of an optional connection pool configuration
+      #    max_connections - The maximum number of connections the connection pool
+      #
+      # examples of vendor-specific options can be found in this
+      # documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc
+      config :sequel_opts, :validate => :hash, :default => {}
+
+      # Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn,
+      # info and debug. The default value is info.
+      config :sql_log_level, :validate => [ "fatal", "error", "warn", "info", "debug" ], :default => "info"
+
+      # Maximum number of times to try connecting to database
+      config :connection_retry_attempts, :validate => :number, :default => 1
+      # Number of seconds to sleep between connection attempts
+      config :connection_retry_attempts_wait_time, :validate => :number, :default => 0.5
+
+      # give users the ability to force Sequel application side into using local timezone
+      config :plugin_timezone, :validate => ["local", "utc"], :default => "utc"
+    end
+
+    private
+    def jdbc_connect
+      opts = {
+      :user => @jdbc_user,
+      :password => @jdbc_password.nil? ? nil : @jdbc_password.value,
+      :pool_timeout => @jdbc_pool_timeout,
+      :keep_reference => false
+      }.merge(@sequel_opts)
+      retry_attempts = @connection_retry_attempts
+      loop do
+        retry_attempts -= 1
+        begin
+          return Sequel.connect(@jdbc_connection_string, opts=opts)
+        rescue Sequel::PoolTimeout => e
+          if retry_attempts <= 0
+            @logger.error("Failed to connect to database. #{@jdbc_pool_timeout} second timeout exceeded. Tried #{@connection_retry_attempts} times.")
+            raise e
+          else
+            @logger.error("Failed to connect to database. #{@jdbc_pool_timeout} second timeout exceeded. Trying again.")
+          end
+        # rescue Java::JavaSql::SQLException, ::Sequel::Error => e
+        rescue ::Sequel::Error => e
+          if retry_attempts <= 0
+            @logger.error("Unable to connect to database. Tried #{@connection_retry_attempts} times", :error_message => e.message, )
+            raise e
+          else
+            @logger.error("Unable to connect to database. Trying again", :error_message => e.message)
+          end
+        end
+        sleep(@connection_retry_attempts_wait_time)
+      end
+    end
+
+    private
+
+    def load_driver_jars
+      unless @jdbc_driver_library.nil? || @jdbc_driver_library.empty?
+        @jdbc_driver_library.split(",").each do |driver_jar|
+          begin
+            @logger.debug("loading #{driver_jar}")
+            # Use https://github.com/jruby/jruby/wiki/CallingJavaFromJRuby#from-jar-files to make classes from jar
+            # available
+            require driver_jar
+          rescue LoadError => e
+            raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
+          end
+        end
+      end
+    end
+
+    private
+    def open_jdbc_connection
+      require "java"
+      require "sequel"
+      require "sequel/adapters/jdbc"
+
+      Sequel.application_timezone = @plugin_timezone.to_sym
+      if @drivers_loaded.false?
+        begin
+          load_driver_jars
+          Sequel::JDBC.load_driver(@jdbc_driver_class)
+        rescue LogStash::Error => e
+          # raised in load_drivers, e.cause should be the caught Java exceptions
+          raise LogStash::PluginLoadingError, "#{e.message} and #{e.cause.message}"
+        rescue Sequel::AdapterNotFound => e
+          # fix this !!!
+          message = if @jdbc_driver_library.nil?
+            ":jdbc_driver_library is not set, are you sure you included
+                      the proper driver client libraries in your classpath?"
+          else
+            "Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
+          end
+          raise LogStash::PluginLoadingError, "#{e}. #{message}"
+        end
+        @drivers_loaded.make_true
+      end
+      @database = jdbc_connect()
+      @database.extension(:pagination)
+      if @jdbc_default_timezone
+        @database.extension(:named_timezones)
+        @database.timezone = @jdbc_default_timezone
+      end
+      if @jdbc_validate_connection
+        @database.extension(:connection_validator)
+        @database.pool.connection_validation_timeout = @jdbc_validation_timeout
+      end
+      @database.fetch_size = @jdbc_fetch_size unless @jdbc_fetch_size.nil?
+      begin
+        @database.test_connection
+      rescue Java::JavaSql::SQLException => e
+        @logger.warn("Failed test_connection with java.sql.SQLException.", :exception => e)
+      rescue Sequel::DatabaseConnectionError => e
+        @logger.warn("Failed test_connection.", :exception => e)
+        close_jdbc_connection
+
+        #TODO return false and let the plugin raise a LogStash::ConfigurationError
+        raise e
+      end
+
+      @database.sql_log_level = @sql_log_level.to_sym
+      @database.logger = @logger
+
+      @database.extension :identifier_mangling
+
+      if @lowercase_column_names
+        @database.identifier_output_method = :downcase
+      else
+        @database.identifier_output_method = :to_s
+      end
+    end
+
+    public
+    def prepare_jdbc_connection
+      @connection_lock = ReentrantLock.new
+      @drivers_loaded = Concurrent::AtomicBoolean.new
+    end
+
+    public
+    def close_jdbc_connection
+      begin
+        # pipeline restarts can also close the jdbc connection, block until the current executing statement is finished to avoid leaking connections
+        # connections in use won't really get closed
+        @connection_lock.lock
+        @database.disconnect if @database
+      rescue => e
+        @logger.warn("Failed to close connection", :exception => e)
+      ensure
+        @connection_lock.unlock
+      end
+    end
+
+    public
+    def execute_statement
+      success = false
+      @connection_lock.lock
+      open_jdbc_connection
+      begin
+        sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
+        @tracking_column_warning_sent = false
+        @statement_handler.perform_query(@database, @value_tracker.value, @jdbc_paging_enabled, @jdbc_page_size) do |row|
+          sql_last_value = get_column_value(row) if @use_column_value
+          yield extract_values_from(row)
+        end
+        success = true
+      rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError, Java::JavaSql::SQLException => e
+        @logger.warn("Exception when executing JDBC query", :exception => e)
+      else
+        @value_tracker.set_value(sql_last_value)
+      ensure
+        close_jdbc_connection
+        @connection_lock.unlock
+      end
+      return success
+    end
+
+    public
+    def get_column_value(row)
+      if !row.has_key?(@tracking_column.to_sym)
+        if !@tracking_column_warning_sent
+          @logger.warn("tracking_column not found in dataset.", :tracking_column => @tracking_column)
+          @tracking_column_warning_sent = true
+        end
+        # If we can't find the tracking column, return the current value in the ivar
+        @value_tracker.value
+      else
+        # Otherwise send the updated tracking column
+        row[@tracking_column.to_sym]
+      end
+    end
+
+    private
+    #Stringify row keys and decorate values when necessary
+    def extract_values_from(row)
+      Hash[row.map { |k, v| [k.to_s, decorate_value(v)] }]
+    end
+
+    private
+    def decorate_value(value)
+      case value
+      when Time
+        # transform it to LogStash::Timestamp as required by LS
+        LogStash::Timestamp.new(value)
+      when Date, DateTime
+        LogStash::Timestamp.new(value.to_time)
+      else
+        value
+      end
+    end
+  end
+end end end
+
+

data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+
+module LogStash module PluginMixins module Jdbc
+  class StatementHandler
+    def self.build_statement_handler(plugin, logger)
+      klass = plugin.use_prepared_statements ? PreparedStatementHandler : NormalStatementHandler
+      klass.new(plugin, logger)
+    end
+
+    attr_reader :statement, :parameters, :statement_logger
+
+    def initialize(plugin, statement_logger)
+      @statement = plugin.statement
+      @statement_logger = statement_logger
+      post_init(plugin)
+    end
+
+    def build_query(db, sql_last_value)
+      # override in subclass
+    end
+
+    def post_init(plugin)
+      # override in subclass, if needed
+    end
+  end
+
+  class NormalStatementHandler < StatementHandler
+    # Performs the query, respecting our pagination settings, yielding once per row of data
+    # @param db [Sequel::Database]
+    # @param sql_last_value [Integet|DateTime|Time]
+    # @yieldparam row [Hash{Symbol=>Object}]
+    def perform_query(db, sql_last_value, jdbc_paging_enabled, jdbc_page_size)
+      query = build_query(db, sql_last_value)
+      if jdbc_paging_enabled
+        query.each_page(jdbc_page_size) do |paged_dataset|
+          paged_dataset.each do |row|
+            yield row
+          end
+        end
+      else
+        query.each do |row|
+          yield row
+        end
+      end
+    end
+
+    private
+
+    def build_query(db, sql_last_value)
+      parameters[:sql_last_value] = sql_last_value
+      query = db[statement, parameters]
+      statement_logger.log_statement_parameters(statement, parameters, query)
+      query
+    end
+
+    def post_init(plugin)
+      @parameter_keys = ["sql_last_value"] + plugin.parameters.keys
+      @parameters = plugin.parameters.inject({}) do |hash,(k,v)|
+        case v
+        when LogStash::Timestamp
+          hash[k.to_sym] = v.time
+        else
+          hash[k.to_sym] = v
+        end
+        hash
+      end
+    end
+  end
+
+  class PreparedStatementHandler < StatementHandler
+    attr_reader :name, :bind_values_array, :statement_prepared, :prepared
+
+    # Performs the query, ignoring our pagination settings, yielding once per row of data
+    # @param db [Sequel::Database]
+    # @param sql_last_value [Integet|DateTime|Time]
+    # @yieldparam row [Hash{Symbol=>Object}]
+    def perform_query(db, sql_last_value, jdbc_paging_enabled, jdbc_page_size)
+      query = build_query(db, sql_last_value)
+      query.each do |row|
+        yield row
+      end
+    end
+
+    private
+
+    def build_query(db, sql_last_value)
+      @parameters = create_bind_values_hash
+      if statement_prepared.false?
+        prepended = parameters.keys.map{|v| v.to_s.prepend("$").to_sym}
+        @prepared = db[statement, *prepended].prepare(:select, name)
+        statement_prepared.make_true
+      end
+      # under the scheduler the Sequel database instance is recreated each time
+      # so the previous prepared statements are lost, add back
+      if db.prepared_statement(name).nil?
+        db.set_prepared_statement(name, prepared)
+      end
+      bind_value_sql_last_value(sql_last_value)
+      statement_logger.log_statement_parameters(statement, parameters, nil)
+      db.call(name, parameters)
+    end
+
+    def post_init(plugin)
+      # don't log statement count when using prepared statements for now...
+      # needs enhancement to allow user to supply a bindable count prepared statement in settings.
+      @statement_logger.disable_count
+
+      @name = plugin.prepared_statement_name.to_sym
+      @bind_values_array = plugin.prepared_statement_bind_values
+      @parameters = plugin.parameters
+      @statement_prepared = Concurrent::AtomicBoolean.new(false)
+    end
+
+    def create_bind_values_hash
+      hash = {}
+      bind_values_array.each_with_index {|v,i| hash[:"p#{i}"] = v}
+      hash
+    end
+
+    def bind_value_sql_last_value(sql_last_value)
+      parameters.keys.each do |key|
+        value = parameters[key]
+        if value == ":sql_last_value"
+          parameters[key] = sql_last_value
+        end
+      end
+    end
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -0,0 +1,140 @@
+# encoding: utf-8
+require "yaml" # persistence
+
+module LogStash module PluginMixins module Jdbc
+  class ValueTracking
+
+    def self.build_last_value_tracker(plugin)
+      handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_path) : NullFileHandler.new(plugin.last_run_metadata_path)
+      if plugin.record_last_run
+        handler = FileHandler.new(plugin.last_run_metadata_path)
+      end
+      if plugin.clean_run
+        handler.clean
+      end
+
+      if plugin.use_column_value && plugin.tracking_column_type == "numeric"
+        # use this irrespective of the jdbc_default_timezone setting
+        NumericValueTracker.new(handler)
+      else
+        if plugin.jdbc_default_timezone.nil? || plugin.jdbc_default_timezone.empty?
+          # no TZ stuff for Sequel, use Time
+          TimeValueTracker.new(handler)
+        else
+          # Sequel does timezone handling on DateTime only
+          DateTimeValueTracker.new(handler)
+        end
+      end
+    end
+
+    attr_reader :value
+
+    def initialize(handler)
+      @file_handler = handler
+      set_initial
+    end
+
+    def set_initial
+      # override in subclass
+    end
+
+    def set_value(value)
+      # override in subclass
+    end
+
+    def write
+      @file_handler.write(@value)
+    end
+
+    private
+    def common_set_initial(method_symbol, default)
+      persisted = @file_handler.read
+
+      if persisted && persisted.respond_to?(method_symbol)
+        @value = persisted
+      else
+        @file_handler.clean
+        @value = default
+      end
+    end
+  end
+
+
+  class NumericValueTracker < ValueTracking
+    def set_initial
+      common_set_initial(:gcd, 0)
+    end
+
+    def set_value(value)
+      return unless value.is_a?(Numeric)
+      @value = value
+    end
+  end
+
+  class DateTimeValueTracker < ValueTracking
+    def set_initial
+      common_set_initial(:to_datetime, DateTime.new(1970))
+    end
+
+    def set_value(value)
+      if value.respond_to?(:to_datetime)
+        @value = value.to_datetime
+      else
+        @value = DateTime.parse(value)
+      end
+    end
+  end
+
+  class TimeValueTracker < ValueTracking
+    def set_initial
+      common_set_initial(:to_time, Time.at(0).utc)
+    end
+
+    def set_value(value)
+      if value.respond_to?(:to_time)
+        @value = value.to_time
+      else
+        @value = DateTime.parse(value).to_time
+      end
+    end
+  end
+
+  class FileHandler
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+      @exists = ::File.exist?(@path)
+    end
+
+    def clean
+      return unless @exists
+      ::File.delete(@path)
+      @exists = false
+    end
+
+    def read
+      return unless @exists
+      YAML.load(::File.read(@path))
+    end
+
+    def write(value)
+      ::File.write(@path, YAML.dump(value))
+      @exists = true
+    end
+  end
+
+  class NullFileHandler
+    def initialize(path)
+    end
+
+    def clean
+    end
+
+    def read
+    end
+
+    def write(value)
+    end
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc_streaming/cache_payload.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+module LogStash module PluginMixins module JdbcStreaming
+  class CachePayload
+    attr_reader :payload
+
+    def initialize
+      @failure = false
+      @payload = []
+    end
+
+    def push(data)
+      @payload << data
+    end
+
+    def failed!
+      @failure = true
+    end
+
+    def failed?
+      @failure
+    end
+
+    def empty?
+      @payload.empty?
+    end
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc_streaming/parameter_handler.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+
+module LogStash module PluginMixins module JdbcStreaming
+  class ParameterHandler
+
+    def self.build_parameter_handler(given_value)
+      # does it really make sense to deal with normal parameters differently?
+      handler = FieldParameter.new(given_value)
+      return handler unless given_value.is_a?(String)
+
+      first_percent_curly = given_value.index("%{")
+      if first_percent_curly && given_value.index("}", first_percent_curly)
+        return InterpolatedParameter.new(given_value)
+      end
+
+      handler
+    end
+
+    def self.build_bind_value_handler(given_value)
+      handler = ConstantParameter.new(given_value)
+
+      return handler unless given_value.is_a?(String) # allow non String constants
+
+      first_percent_curly = given_value.index("%{")
+      if first_percent_curly && given_value.index("}", first_percent_curly)
+        return InterpolatedParameter.new(given_value)
+      end
+
+      if given_value =~ /\A\s*\[[^\]]+\]\s*\z/
+        return FieldParameter.new(given_value)
+      end
+
+      handler
+    end
+
+    attr_reader :given_value
+
+    def initialize(given_value)
+      @given_value = given_value
+    end
+
+    def extract_from(event)
+      # override in subclass
+    end
+  end
+
+  class InterpolatedParameter < ParameterHandler
+    def extract_from(event)
+      event.sprintf(@given_value)
+    end
+  end
+
+  class FieldParameter < ParameterHandler
+    def extract_from(event)
+      event.get(@given_value)
+    end
+  end
+
+  class ConstantParameter < ParameterHandler
+    def extract_from(event)
+      @given_value
+    end
+  end
+end end end