logstash-filter-jdbc_static 1.0.0

data/lib/logstash/filters/jdbc/loader_schedule.rb

@@ -0,0 +1,38 @@
+require_relative "validatable"
+
+module LogStash module Filters module Jdbc
+  class LoaderSchedule < Validatable
+    attr_reader :schedule_frequency, :loader_schedule
+
+    private
+
+    def post_initialize
+      if valid?
+        # From the Rufus::Scheduler docs:
+        # By default, rufus-scheduler sleeps 0.300 second between every step.
+        # At each step it checks for jobs to trigger and so on.
+        if @cronline.seconds.is_a?(Set)
+          @schedule_frequency = 0.3
+        else
+          @schedule_frequency = 30
+        end
+      end
+    end
+
+    def parse_options
+      @loader_schedule = @options
+
+      unless @loader_schedule.is_a?(String)
+        @option_errors << "The loader_schedule option must be a string"
+      end
+
+      begin
+        @cronline = Rufus::Scheduler::CronLine.new(@loader_schedule)
+      rescue => e
+        @option_errors << "The loader_schedule option is invalid: #{e.message}"
+      end
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/lookup.rb

@@ -0,0 +1,192 @@
+require_relative "lookup_result"
+require "logstash/util/loggable"
+
+module LogStash module Filters module Jdbc
+  class Lookup
+    include LogStash::Util::Loggable
+
+    class Sprintfier
+      def initialize(param)
+        @param = param
+      end
+
+      def fetch(event, result)
+        formatted = event.sprintf(@param)
+        if formatted == @param # no field found so no transformation
+          result.invalid_parameters_push(@param)
+        end
+        formatted
+      end
+    end
+
+    class Getfier
+      def initialize(param)
+        @param = param
+      end
+
+      def fetch(event, result)
+        value = event.get(@param)
+        if value.nil? || value.is_a?(Hash) || value.is_a?(Array) # Array or Hash is not suitable
+          result.invalid_parameters_push(@param)
+        end
+        value
+      end
+    end
+
+    def self.find_validation_errors(array_of_options)
+      if !array_of_options.is_a?(Array)
+        return "The options must be an Array"
+        end
+      errors = []
+      array_of_options.each_with_index do |options, i|
+        instance = new(options, {}, "lookup-#{i.next}")
+        unless instance.valid?
+          errors << instance.formatted_errors
+        end
+      end
+      return nil if errors.empty?
+      errors.join("; ")
+    end
+
+    attr_reader :id, :target, :query, :parameters
+
+    def initialize(options, globals, default_id)
+      @id = options["id"] || default_id
+      @target = options["target"]
+      @id_used_as_target = @target.nil?
+      if @id_used_as_target
+        @target = @id
+      end
+      @options = options
+      @globals = globals
+      @valid = false
+      @option_errors = []
+      @default_result = nil
+      parse_options
+    end
+
+    def id_used_as_target?
+      @id_used_as_target
+    end
+
+    def valid?
+      @valid
+    end
+
+    def formatted_errors
+      @option_errors.join(", ")
+    end
+
+    def enhance(local, event)
+      result = fetch(local, event) # should return a LookupResult
+
+      if result.failed? || result.parameters_invalid?
+        tag_failure(event)
+      end
+
+      if result.valid?
+        if @use_default && result.empty?
+          tag_default(event)
+          process_event(event, @default_result)
+        else
+          process_event(event, result)
+        end
+        true
+      else
+        false
+      end
+    end
+
+    private
+
+    def tag_failure(event)
+      @tag_on_failure.each do |tag|
+        event.tag(tag)
+      end
+    end
+
+    def tag_default(event)
+      @tag_on_default_use.each do |tag|
+        event.tag(tag)
+      end
+    end
+
+    def fetch(local, event)
+      result = LookupResult.new()
+      if @parameters_specified
+        params = prepare_parameters_from_event(event, result)
+        if result.parameters_invalid?
+          logger.warn? && logger.warn("Parameter field not found in event", :lookup_id => @id, :invalid_parameters => result.invalid_parameters)
+          return result
+        end
+      else
+        params = {}
+      end
+      begin
+        logger.debug? && logger.debug("Executing Jdbc query", :lookup_id => @id, :statement => query, :parameters => params)
+        local.fetch(query, params).each do |row|
+          stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
+          result.push(stringified)
+        end
+      rescue ::Sequel::Error => e
+        # all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
+        result.failed!
+        logger.warn? && logger.warn("Exception when executing Jdbc query", :lookup_id => @id, :exception => e.message, :backtrace => e.backtrace.take(8))
+      end
+      # if either of: no records or a Sequel exception occurs the payload is
+      # empty and the default can be substituted later.
+      result
+    end
+
+    def process_event(event, result)
+      # use deep clone here so other filter function don't taint the payload by reference
+      event.set(@target, ::LogStash::Util.deep_clone(result.payload))
+    end
+
+    def prepare_parameters_from_event(event, result)
+      @symbol_parameters.inject({}) do |hash,(k,v)|
+        value = v.fetch(event, result)
+        hash[k] = value.is_a?(::LogStash::Timestamp) ? value.time : value
+        hash
+      end
+    end
+
+    def sprintf_or_get(v)
+      v.match(/%{([^}]+)}/) ? Sprintfier.new(v) : Getfier.new(v)
+    end
+
+    def parse_options
+      @query = @options["query"]
+      unless @query && @query.is_a?(String)
+        @option_errors << "The options for '#{@id}' must include a 'query' string"
+      end
+
+      @parameters = @options["parameters"]
+      @parameters_specified = false
+      if @parameters
+        if !@parameters.is_a?(Hash)
+          @option_errors << "The 'parameters' option for '#{@id}' must be a Hash"
+        else
+          # this is done once per lookup at start, i.e. Sprintfier.new et.al is done once.
+          @symbol_parameters = @parameters.inject({}) {|hash,(k,v)| hash[k.to_sym] = sprintf_or_get(v) ; hash }
+          # the user might specify an empty hash parameters => {}
+          # maybe due to an unparameterised query
+          @parameters_specified = !@symbol_parameters.empty?
+        end
+      end
+
+      default_hash = @options["default_hash"]
+      if default_hash && !default_hash.empty?
+        @default_result = LookupResult.new()
+        @default_result.push(default_hash)
+      end
+
+      @use_default = !@default_result.nil?
+
+      @tag_on_failure = @options["tag_on_failure"] || @globals["tag_on_failure"] || []
+      @tag_on_default_use = @options["tag_on_default_use"] || @globals["tag_on_default_use"] || []
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/lookup_processor.rb

@@ -0,0 +1,91 @@
+require_relative "lookup"
+require_relative "read_write_database"
+
+module LogStash module Filters module Jdbc
+  class LookupProcessor
+    attr_reader :lookups, :local
+
+    CONNECTION_ERROR_MSG = "Connection error when initialising lookup (local) db"
+    DISCONNECTION_ERROR_MSG = "Connection error when disconnecting from lookup (local) db"
+
+    def self.find_validation_errors(array_of_options)
+      if !array_of_options.is_a?(Array)
+        return "The options must be an Array"
+      end
+      errors = []
+      instance = new(array_of_options, {})
+      instance.lookups.each do |lookup|
+        unless lookup.valid?
+          errors << lookup.formatted_errors
+        end
+      end
+      unless instance.valid?
+        errors << instance.formatted_errors
+      end
+      return nil if errors.empty?
+      # errors.unshift("For plugin #{}")
+      errors.join("; ")
+    end
+
+    def initialize(lookups_array, globals)
+      @lookups_errors = []
+      @lookups = lookups_array.map.with_index do |options, i|
+        Lookup.new(options, globals, "lookup-#{i.next}")
+      end
+      validate_lookups
+      if @lookups_errors.empty? && !globals.empty?
+        @local = ReadWriteDatabase.create(*globals.values_at(
+          "lookup_jdbc_connection_string",
+          "lookup_jdbc_driver_class",
+          "lookup_jdbc_driver_library").compact)
+        @local.connect(CONNECTION_ERROR_MSG)
+      end
+    end
+
+    def enhance(event)
+      @lookups.map { |lookup| lookup.enhance(@local, event) }
+    end
+
+    def close
+      @local.disconnect(DISCONNECTION_ERROR_MSG)
+      @local = nil
+    end
+
+    def formatted_errors
+      @lookups_errors.join(", ")
+    end
+
+    def valid?
+      @lookups_errors.empty?
+    end
+
+    private
+
+    def validate_lookups
+      ids = Hash.new(0)
+      errors = []
+      @lookups.each {|lookup| ids[lookup.id] += 1}
+      ids.select{|id, count| count > 1}.each do |id, count|
+        errors << "'#{id}' is specified multiple times"
+      end
+      if !errors.empty?
+        errors.unshift("Id setting must be different across all lookups")
+      end
+      @lookups_errors.concat(errors)
+      targets = Hash.new {|h,k| h[k] = []}
+      errors = []
+      @lookups.each do |lookup|
+        # if id was used as target, skip target unique check because id uniqueness is checked already
+        next if lookup.id_used_as_target?
+        targets[lookup.target] << lookup.id
+      end
+      targets.select{|_,val| val.size > 1}.each do |target, ids|
+        errors << "'#{ids.join("', '")}' have the same target field setting"
+      end
+      if !errors.empty?
+        errors.unshift("Target setting must be different across all lookups")
+      end
+      @lookups_errors.concat(errors)
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/lookup_result.rb

@@ -0,0 +1,39 @@
+module LogStash module Filters module Jdbc
+  class LookupResult
+    attr_reader :payload, :invalid_parameters
+
+    def initialize
+      @failure = false
+      @payload = []
+      @invalid_parameters = []
+    end
+
+    def push(data)
+      @payload << data
+    end
+
+    def invalid_parameters_push(data)
+      @invalid_parameters << data
+    end
+
+    def failed!
+      @failure = true
+    end
+
+    def valid?
+      !failed? && @invalid_parameters.empty?
+    end
+
+    def failed?
+      @failure
+    end
+
+    def parameters_invalid?
+      !@invalid_parameters.empty?
+    end
+
+    def empty?
+      @payload.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/read_only_database.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require_relative "basic_database"
+
+module LogStash module Filters module Jdbc
+  class ReadOnlyDatabase < BasicDatabase
+
+    def count(statement)
+      result = 0
+      debug_log_messages = ["Lookup query count is zero"]
+      begin
+        # its the responsibility of the caller to manage the connections see Loader
+        if connected?
+          result = @db[statement].count
+        else
+          debug_log_messages.concat("and there is no connection to the remote db at this time")
+        end
+      rescue ::Sequel::Error => err
+        # a fatal issue
+        msg = "Exception occurred when executing loader Jdbc query count"
+        logger.error(msg, :exception => err.message, :backtrace => err.backtrace.take(8))
+        raise wrap_error(LookupJdbcException, err, msg)
+      end
+      logger.debug(debug_log_messages.join(' ')) if result.zero?
+      result
+    end
+
+    def query(statement)
+      result = empty_record_set
+      debug_log_messages = ["Lookup query results are empty"]
+      begin
+        # its the responsibility of the caller to manage the connections see Loader
+        if connected?
+          result = @db[statement].all
+        else
+          debug_log_messages.concat("and there is no connection to the remote db at this time")
+        end
+      rescue ::Sequel::Error => err
+        # a fatal issue
+        msg = "Exception occurred when executing loader Jdbc query"
+        logger.error(msg, :exception => err.message, :backtrace => err.backtrace.take(8))
+        raise wrap_error(LookupJdbcException, err, msg)
+      end
+      logger.debug(debug_log_messages.join(' ')) if result.empty?
+      result
+    end
+
+    def post_create(connection_string, driver_class, driver_library, user, password)
+      verify_connection(connection_string, driver_class, driver_library, user, password)
+    end
+
+    private
+
+    def post_initialize()
+      super
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/read_write_database.rb

@@ -0,0 +1,86 @@
+require_relative "basic_database"
+
+module LogStash module Filters module Jdbc
+  class ReadWriteDatabase < BasicDatabase
+    def repopulate_all(loaders)
+      case loaders.size
+        when 1
+          fill_and_switch(loaders.first)
+        when 2
+          fill_and_switch(loaders.first)
+          fill_and_switch(loaders.last)
+        else
+          loaders.each do |loader|
+            fill_and_switch(loader)
+          end
+      end
+    end
+
+    alias populate_all repopulate_all
+
+    def fetch(statement, parameters)
+      @rwlock.readLock().lock()
+      # any exceptions should bubble up because we need to set failure tags etc.
+      @db[statement, parameters].all
+    ensure
+      @rwlock.readLock().unlock()
+    end
+
+    def build_db_object(db_object)
+      begin
+        @rwlock.writeLock().lock()
+        db_object.build(@db)
+      rescue *CONNECTION_ERRORS => err
+        # we do not raise an error when there is a connection error, we hope that the connection works next time
+        logger.error("Connection error when initialising lookup db", :db_object => db_object.inspect, :exception => err.message, :backtrace => err.backtrace.take(8))
+      rescue ::Sequel::Error => err
+        msg = "Exception when initialising lookup db for db object: #{db_object}"
+        logger.error(msg, :exception => err.message, :backtrace => err.backtrace.take(8))
+        raise wrap_error(LoaderJdbcException, err, msg)
+      ensure
+        @rwlock.writeLock().unlock()
+      end
+    end
+
+    def post_create(connection_string, driver_class, driver_library, user, password)
+      mutated_connection_string = connection_string.sub("____", unique_db_name)
+      verify_connection(mutated_connection_string, driver_class, driver_library, user, password)
+      connect("Connection error when connecting to lookup db")
+    end
+
+    private
+
+    def fill_and_switch(loader)
+      begin
+        records = loader.fetch
+        return if records.size.zero?
+        @rwlock.writeLock().lock()
+        tmp = self.class.random_name
+        @db.transaction do |conn|
+          @db[loader.temp_table].multi_insert(records)
+          @db.rename_table(loader.temp_table, tmp)
+          @db.rename_table(loader.table, loader.temp_table)
+          @db.rename_table(tmp, loader.table)
+          @db[loader.temp_table].truncate
+        end
+      rescue *CONNECTION_ERRORS => err
+        # we do not raise an error when there is a connection error, we hope that the connection works next time
+        logger.error("Connection error when filling lookup db from loader query results", :exception => err.message, :backtrace => err.backtrace.take(8))
+      rescue => err
+        # In theory all exceptions in Sequel should be wrapped in Sequel::Error
+        # There are cases where exceptions occur in unprotected ensure sections
+        msg = "Exception when filling lookup db from loader query results, original exception: #{err.class}, original message: #{err.message}"
+        logger.error(msg, :backtrace => err.backtrace.take(16))
+        raise wrap_error(LoaderJdbcException, err, msg)
+      ensure
+        @rwlock.writeLock().unlock()
+      end
+    end
+
+    def post_initialize()
+      super
+      # get a fair reentrant read write lock
+      @rwlock = java.util.concurrent.locks.ReentrantReadWriteLock.new(true)
+    end
+  end
+end end end