logstash-integration-jdbc 5.0.0.alpha1

data/lib/logstash/filters/jdbc/lookup_result.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+module LogStash module Filters module Jdbc
+  class LookupResult
+    attr_reader :payload, :invalid_parameters
+
+    def initialize
+      @failure = false
+      @payload = []
+      @invalid_parameters = []
+    end
+
+    def push(data)
+      @payload << data
+    end
+
+    def invalid_parameters_push(data)
+      @invalid_parameters << data
+    end
+
+    def failed!
+      @failure = true
+    end
+
+    def valid?
+      !failed? && @invalid_parameters.empty?
+    end
+
+    def failed?
+      @failure
+    end
+
+    def parameters_invalid?
+      !@invalid_parameters.empty?
+    end
+
+    def empty?
+      @payload.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/read_only_database.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require_relative "basic_database"
+
+module LogStash module Filters module Jdbc
+  class ReadOnlyDatabase < BasicDatabase
+
+    def count(statement)
+      result = 0
+      debug_log_messages = ["Lookup query count is zero"]
+      begin
+        # its the responsibility of the caller to manage the connections see Loader
+        if connected?
+          result = @db[statement].count
+        else
+          debug_log_messages.concat("and there is no connection to the remote db at this time")
+        end
+      rescue ::Sequel::Error => err
+        # a fatal issue
+        msg = "Exception occurred when executing loader Jdbc query count"
+        logger.error(msg, :exception => err.message, :backtrace => err.backtrace.take(8))
+        raise wrap_error(LookupJdbcException, err, msg)
+      end
+      logger.debug(debug_log_messages.join(' ')) if result.zero?
+      result
+    end
+
+    def query(statement)
+      result = empty_record_set
+      debug_log_messages = ["Lookup query results are empty"]
+      begin
+        # its the responsibility of the caller to manage the connections see Loader
+        if connected?
+          result = @db[statement].all
+        else
+          debug_log_messages.concat("and there is no connection to the remote db at this time")
+        end
+      rescue ::Sequel::Error => err
+        # a fatal issue
+        msg = "Exception occurred when executing loader Jdbc query"
+        logger.error(msg, :exception => err.message, :backtrace => err.backtrace.take(8))
+        raise wrap_error(LookupJdbcException, err, msg)
+      end
+      logger.debug(debug_log_messages.join(' ')) if result.empty?
+      result
+    end
+
+    def post_create(connection_string, driver_class, driver_library, user, password)
+      verify_connection(connection_string, driver_class, driver_library, user, password)
+    end
+
+    private
+
+    def post_initialize()
+      super
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/read_write_database.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+require_relative "basic_database"
+
+module LogStash module Filters module Jdbc
+  class ReadWriteDatabase < BasicDatabase
+    def repopulate_all(loaders)
+      case loaders.size
+        when 1
+          fill_local_table(loaders.first)
+        when 2
+          fill_local_table(loaders.first)
+          fill_local_table(loaders.last)
+        else
+          loaders.each do |loader|
+            fill_local_table(loader)
+          end
+      end
+    end
+
+    alias populate_all repopulate_all
+
+    def fetch(statement, parameters)
+      @rwlock.readLock().lock()
+      # any exceptions should bubble up because we need to set failure tags etc.
+      @db[statement, parameters].all
+    ensure
+      @rwlock.readLock().unlock()
+    end
+
+    def prepare(statement, parameters)
+      @rwlock.readLock().lock()
+      @db[statement, parameters].prepare(:select, @id)
+    ensure
+      @rwlock.readLock().unlock()
+    end
+
+    def build_db_object(db_object)
+      begin
+        @rwlock.writeLock().lock()
+        db_object.build(@db)
+        if db_object.index_columns.empty?
+          logger.warn("local_db_object '#{db_object.name}': `index_columns` is optional but on larger datasets consider adding an index on the lookup column, it will improve performance")
+        end
+      rescue *CONNECTION_ERRORS => err
+        # we do not raise an error when there is a connection error, we hope that the connection works next time
+        logger.error("Connection error when initialising lookup db", :db_object => db_object.inspect, :exception => err.message, :backtrace => err.backtrace.take(8))
+      rescue ::Sequel::Error => err
+        msg = "Exception when initialising lookup db for db object: #{db_object}"
+        logger.error(msg, :exception => err.message, :backtrace => err.backtrace.take(8))
+        raise wrap_error(LoaderJdbcException, err, msg)
+      ensure
+        @rwlock.writeLock().unlock()
+      end
+    end
+
+    def post_create(connection_string, driver_class, driver_library, user, password)
+      mutated_connection_string = connection_string.sub("____", unique_db_name)
+      verify_connection(mutated_connection_string, driver_class, driver_library, user, password)
+      connect("Connection error when connecting to lookup db")
+    end
+
+    private
+
+    def fill_local_table(loader)
+      begin
+        @rwlock.writeLock().lock()
+        start = Time.now.to_f
+        records = loader.fetch
+        records_size = records.size
+        return if records_size.zero?
+        logger.info("loader #{loader.id}, fetched #{records_size} records in: #{(Time.now.to_f - start).round(3)} seconds")
+        start = Time.now.to_f
+        import_file = ::File.join(loader.staging_directory, loader.table.to_s)
+        ::File.open(import_file, "w") do |fd|
+          dataset = @db[loader.table]
+          records.each do |hash|
+            array = hash.values.map {|val| dataset.literal(val) }
+            fd.puts(array.join(", "))
+          end
+          fd.fsync
+        end
+        logger.info("loader #{loader.id}, saved fetched records to import file in: #{(Time.now.to_f - start).round(3)} seconds")
+        start = Time.now.to_f
+        import_cmd = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE (null,'#{loader.table.upcase}','#{import_file}',null,'''',null,1)"
+        @db.execute_ddl(import_cmd)
+        FileUtils.rm_f(import_file)
+        logger.info("loader #{loader.id}, imported all fetched records in: #{(Time.now.to_f - start).round(3)} seconds")
+      rescue *CONNECTION_ERRORS => err
+        # we do not raise an error when there is a connection error, we hope that the connection works next time
+        logger.error("Connection error when filling lookup db from loader #{loader.id}, query results", :exception => err.message, :backtrace => err.backtrace.take(8))
+      rescue => err
+        # In theory all exceptions in Sequel should be wrapped in Sequel::Error
+        # There are cases where exceptions occur in unprotected ensure sections
+        msg = "Exception when filling lookup db from loader #{loader.id}, query results, original exception: #{err.class}, original message: #{err.message}"
+        logger.error(msg, :backtrace => err.backtrace.take(16))
+        raise wrap_error(LoaderJdbcException, err, msg)
+      ensure
+        @rwlock.writeLock().unlock()
+      end
+    end
+
+    def post_initialize()
+      super
+      # get a fair reentrant read write lock
+      @rwlock = java.util.concurrent.locks.ReentrantReadWriteLock.new(true)
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/repeating_load_runner.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+require_relative "single_load_runner"
+
+module LogStash module Filters module Jdbc
+  class RepeatingLoadRunner < SingleLoadRunner
+   # info - attr_reader :local, :loaders, :preloaders
+
+    def repeated_load
+      local.repopulate_all(loaders)
+      @reload_counter.increment
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/single_load_runner.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+require_relative 'db_object'
+
+module LogStash module Filters module Jdbc
+  class SingleLoadRunner
+
+    attr_reader :local, :loaders, :preloaders
+
+    def initialize(local, loaders, preloaders)
+      @local = local
+      @loaders = loaders
+      @preloaders = []
+      @reload_counter = Concurrent::AtomicFixnum.new(0)
+      preloaders.map do |pre|
+        @preloaders << DbObject.new(pre)
+      end
+      @preloaders.sort!
+    end
+
+    def initial_load
+      do_preload
+      local.populate_all(loaders)
+      @reload_counter.increment
+    end
+
+    def repeated_load
+    end
+
+    def call
+      repeated_load
+    end
+
+    def reload_count
+      @reload_counter.value
+    end
+
+    private
+
+    def do_preload
+      preloaders.each do |db_object|
+        local.build_db_object(db_object)
+      end
+    end
+  end
+
+end end end

data/lib/logstash/filters/jdbc/validatable.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+module LogStash module Filters module Jdbc
+  class Validatable
+    def self.find_validation_errors(options)
+      array_of_options = Array(options)
+      errors = []
+      array_of_options.each do |options|
+        instance = new(options)
+        unless instance.valid?
+          errors << instance.formatted_errors
+        end
+      end
+      return nil if errors.empty?
+      errors.join("; ")
+    end
+
+    def initialize(options)
+      pre_initialize(options)
+      @options = options
+      @valid = false
+      @option_errors = []
+      parse_options
+      post_initialize
+    end
+
+    def valid?
+      @valid
+    end
+
+    def formatted_errors
+      @option_errors.join(", ")
+    end
+
+    private
+
+    def pre_initialize(options)
+    end
+
+    def post_initialize
+    end
+
+    def parse_options
+      raise "Subclass must implement 'parse_options'"
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc_static.rb

@@ -0,0 +1,240 @@
+# encoding: utf-8
+require "logstash-integration-jdbc_jars"
+require "logstash/filters/base"
+require "logstash/namespace"
+require_relative "jdbc/loader"
+require_relative "jdbc/loader_schedule"
+require_relative "jdbc/repeating_load_runner"
+require_relative "jdbc/lookup_processor"
+
+# This filter can do multiple enhancements to an event in one pass.
+# Define multiple loader sources and multiple lookup targets.
+# Currently only one remote database connection is supported.
+# [source,ruby]
+
+#
+module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
+  config_name "jdbc_static"
+
+  # Define the loaders, an Array of Hashes, to fetch remote data and create local tables.
+  # the fetched data will be inserted into the local tables. Make sure that the
+  # local table name, columns and datatypes correspond to the shape of the remote data
+  # being fetched. The default for max_rows is 1 million rows. You may provide an `id`
+  # For example:
+  # loaders => [
+  #   {
+  #     id => "country_details"
+  #     query => "select code, name from WORLD.COUNTRY"
+  #     max_rows => 2000
+  #     local_table => "country"
+  #   },
+  #   {
+  #     id => "servers_load"
+  #     query => "select id, ip, name, location from INTERNAL.SERVERS"
+  #     local_table => "servers"
+  #   }
+  # ]
+  # This is optional. You can provide a pre-populated local database server then no initial loaders are needed.
+  config :loaders, :required => false, :default => [], :validate => [LogStash::Filters::Jdbc::Loader]
+
+  # Define an array of Database Objects to create when the plugin first starts.
+  # These will usually be the definitions to setup the local in-memory tables.
+  # For example:
+  # local_db_objects => [
+  #   {name => "servers", preserve_existing => true, index_columns => ["ip"], columns => [["id", "INTEGER"], ["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+  # ]
+  # NOTE: Important! use `preserve_existing => true` to keep a table created and filled in a previous Logstash session. It will default to false and is unneeded if the database is not persistent.
+  # NOTE: Important! Tables created here must have the same names as those used in the `loaders` and
+  # `local_lookups` configuration options
+  config :local_db_objects, :required => false, :default => [], :validate => [LogStash::Filters::Jdbc::DbObject]
+
+  # Define the list (Array) of enhancement local_lookups to be applied to an event
+  # Each entry is a hash of the query string, the target field and value and a
+  # parameter hash. Target is overwritten if existing. Target is optional,
+  # if omitted the lookup results will be written to the root of the event like this:
+  # event.set(<column name (or alias)>, <column value>)
+  # Use parameters to have this plugin put values from the event into the query.
+  # The parameter maps the symbol used in the query string to the field name in the event.
+  # NOTE: when using a query string that includes the LIKE keyword make sure that
+  # you provide a Logstash Event sprintf pattern with added wildcards.
+  # For example:
+  # local_lookups => [
+  #   {
+  #     "query" => "SELECT * FROM country WHERE code = :code",
+  #     "parameters" => {"code" => "country_code"}
+  #     "target" => "country_details"
+  #   },
+  #   {
+  #     "query" => "SELECT ip, name FROM servers WHERE ip LIKE :ip",
+  #     "parameters" => {"ip" => "%{[response][ip]}%"}
+  #     "target" => "servers"
+  #   },
+  #   {
+  #     "query" => "SELECT ip, name FROM servers WHERE ip = ?",
+  #     "prepared_parameters" => ["from_ip"]
+  #     "target" => "servers"
+  #   }
+  # ]
+  config :local_lookups, :required => true, :validate => [LogStash::Filters::Jdbc::LookupProcessor]
+
+  # Schedule of when to periodically run loaders, in Cron format
+  # for example: "* * * * *" (execute query every minute, on the minute)
+  #
+  # There is no schedule by default. If no schedule is given, then the loaders are run
+  # exactly once.
+  config :loader_schedule, :required => false, :validate => LogStash::Filters::Jdbc::LoaderSchedule
+
+  # Append values to the `tags` field if sql error occured
+  # Alternatively, individual `tag_on_failure` arrays can be added to each lookup hash
+  config :tag_on_failure, :validate => :array, :default => ["_jdbcstaticfailure"]
+
+  # Append values to the `tags` field if no record was found and default values were used
+  config :tag_on_default_use, :validate => :array, :default => ["_jdbcstaticdefaultsused"]
+
+  # Remote Load DB Jdbc driver library path to third party driver library.
+  # Use comma separated paths in one string if you need more than one library.
+  config :jdbc_driver_library, :validate => :string
+
+  # Remote Load DB Jdbc driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver"
+  config :jdbc_driver_class, :validate => :string, :required => true
+
+  # Remote Load DB Jdbc connection string
+  config :jdbc_connection_string, :validate => :string, :required => true
+
+  # Remote Load DB Jdbc user
+  config :jdbc_user, :validate => :string
+
+  # Remote Load DB Jdbc password
+  config :jdbc_password, :validate => :password
+
+  # directory for temp files created during bulk loader import.
+  config :staging_directory, :validate => :string, :default => ::File.join(Dir.tmpdir, "logstash", config_name, "import_data")
+
+  # NOTE: For the initial release, we are not allowing the user to specify their own local lookup JDBC DB settings.
+  # In the near future we have to consider identical config running in multiple pipelines stomping over each other
+  # when the database names are common across configs because there is only one Derby server in memory per JVM.
+
+  # Local Lookup DB Jdbc driver class to load, for example "org.apache.derby.jdbc.ClientDriver"
+  # config :lookup_jdbc_driver_class, :validate => :string, :required => false
+
+  # Local Lookup DB Jdbc driver library path to third party driver library.
+  # config :lookup_jdbc_driver_library, :validate => :path, :required => false
+
+  # Local Lookup DB Jdbc connection string
+  # config :lookup_jdbc_connection_string, :validate => :string, :required => false
+
+  class << self
+    alias_method :old_validate_value, :validate_value
+
+    def validate_value(value, validator)
+      if validator.is_a?(Array) && validator.first.respond_to?(:find_validation_errors)
+        validation_errors = validator.first.find_validation_errors(value)
+        unless validation_errors.nil?
+          return false, validation_errors
+        end
+      elsif validator.respond_to?(:find_validation_errors)
+        validation_errors = validator.find_validation_errors(value)
+        unless validation_errors.nil?
+          return false, validation_errors
+        end
+      else
+        return old_validate_value(value, validator)
+      end
+      [true, value]
+    end
+  end
+
+  public
+
+  def register
+    prepare_data_dir
+    prepare_runner
+  end
+
+  def filter(event)
+    enhancement_states = @processor.enhance(event)
+    filter_matched(event) if enhancement_states.all?
+  end
+
+  def close
+    @scheduler.stop if @scheduler
+    @parsed_loaders.each(&:close)
+    @processor.close
+  end
+
+  def loader_runner
+    # use for test verification
+    @loader_runner
+  end
+
+  private
+
+  def prepare_data_dir
+    # later, when local persistent databases are allowed set this property to LS_HOME/data/jdbc-static/
+    # must take multi-pipelines into account and more than one config using the same jdbc-static settings
+    java.lang.System.setProperty("derby.system.home", ENV["HOME"])
+    logger.info("derby.system.home is: #{java.lang.System.getProperty("derby.system.home")}")
+  end
+
+  def prepare_runner
+    @parsed_loaders = @loaders.map do |options|
+      add_plugin_configs(options)
+      loader = Jdbc::Loader.new(options)
+      loader.build_remote_db
+      loader
+    end
+    runner_args = [@parsed_loaders, @local_db_objects]
+    @processor = Jdbc::LookupProcessor.new(@local_lookups, global_lookup_options)
+    runner_args.unshift(@processor.local)
+    if @loader_schedule
+      args = []
+      @loader_runner = Jdbc::RepeatingLoadRunner.new(*runner_args)
+      @loader_runner.initial_load
+      cronline = Jdbc::LoaderSchedule.new(@loader_schedule)
+      cronline.to_log_string.tap do |msg|
+        logger.info("Scheduler operations: #{msg}") unless msg.empty?
+      end
+      logger.info("Scheduler scan for work frequency is: #{cronline.schedule_frequency}")
+      rufus_args = {:max_work_threads => 1, :frequency => cronline.schedule_frequency}
+      @scheduler = Rufus::Scheduler.new(rufus_args)
+      @scheduler.cron(cronline.loader_schedule, @loader_runner)
+    else
+      @loader_runner = Jdbc::SingleLoadRunner.new(*runner_args)
+      @loader_runner.initial_load
+    end
+  end
+
+  def global_lookup_options(options = Hash.new)
+    if @tag_on_failure && !@tag_on_failure.empty? && !options.key?("tag_on_failure")
+      options["tag_on_failure"] = @tag_on_failure
+    end
+    if @tag_on_default_use && !@tag_on_default_use.empty? && !options.key?("tag_on_default_use")
+      options["tag_on_default_use"] = @tag_on_default_use
+    end
+    options["lookup_jdbc_driver_class"] = @lookup_jdbc_driver_class
+    options["lookup_jdbc_driver_library"] = @lookup_jdbc_driver_library
+    options["lookup_jdbc_connection_string"] = @lookup_jdbc_connection_string
+    options
+  end
+
+  def add_plugin_configs(options)
+    if @jdbc_driver_library
+      options["jdbc_driver_library"] = @jdbc_driver_library
+    end
+    if @jdbc_driver_class
+      options["jdbc_driver_class"] = @jdbc_driver_class
+    end
+    if @jdbc_connection_string
+      options["jdbc_connection_string"] = @jdbc_connection_string
+    end
+    if @jdbc_user
+      options["jdbc_user"] = @jdbc_user
+    end
+    if @jdbc_password
+      options["jdbc_password"] = @jdbc_password
+    end
+    if @staging_directory
+      options["staging_directory"] = @staging_directory
+    end
+  end
+end end end