logstash-integration-jdbc 5.0.0.alpha1

data/lib/logstash/filters/jdbc/column.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+require_relative "validatable"
+
+module LogStash module Filters module Jdbc
+  class Column < Validatable
+    attr_reader :name, :datatype
+
+    private
+
+    def post_initialize
+      if valid?
+        @name = @name.to_sym
+        @datatype = @datatype.to_sym
+      end
+    end
+
+    def to_array
+      [@name.to_s, @datatype.to_s]
+    end
+
+    def parse_options
+      unless @options.is_a?(Array)
+        @option_errors << "The column options must be an array"
+      end
+
+      @name, @datatype = @options
+
+      unless @name && @name.is_a?(String)
+        @option_errors << "The first column option is the name and must be a string"
+      end
+
+      unless @datatype && @datatype.is_a?(String)
+        @option_errors << "The second column option is the datatype and must be a string"
+      end
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/db_object.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+require_relative "validatable"
+require_relative "column"
+
+module LogStash module Filters module Jdbc
+
+  class DbObject < Validatable
+    #   {name => "servers", index_columns => ["ip"], columns => [["ip", "text"], ["name", "text"], ["location", "text"]]},
+
+    attr_reader :name, :columns, :preserve_existing, :index_columns
+
+    def build(db)
+      return unless valid?
+      if db.nil?
+        raise "DbObject given a database instance that is nil"
+      end
+      schema_gen = db.create_table_generator()
+      @columns.each {|col| schema_gen.column(col.name, col.datatype)}
+      schema_gen.index(@index_columns) unless @index_columns.empty?
+      options = {:generator => schema_gen}
+      if @preserve_existing
+        db.create_table?(@name, options)
+      else
+        db.create_table(@name, options)
+      end
+    end
+
+    def <=>(other)
+      @name <=> other.name
+    end
+
+    def to_s
+      inspect
+    end
+
+    def inspect
+      "<LogStash::Filters::Jdbc::DbObject name: #{@name}, columns: #{@columns.inspect}>"
+    end
+
+    private
+
+    def post_initialize
+      if valid?
+        @name = @name.to_sym
+      end
+    end
+
+    def parse_options
+      if !@options.is_a?(Hash)
+        @option_errors << "DbObject options must be a Hash"
+        @valid = false
+        return
+      end
+
+      @name = @options["name"]
+      unless @name && @name.is_a?(String)
+        @option_errors << "DbObject options must include a 'name' string"
+        @name = "unnamed"
+      end
+
+      @preserve_existing = @options.fetch("preserve_existing", false)
+      @preserve_existing = true if @preserve_existing == "true"
+
+      @columns_options = @options["columns"]
+      @columns = []
+      temp_column_names = []
+      if @columns_options && @columns_options.is_a?(Array)
+        sizes = @columns_options.map{|option| option.size}.uniq
+        if sizes == [2]
+          @columns_options.each do |option|
+            column = Column.new(option)
+            if column.valid?
+              @columns << column
+              temp_column_names << column.name
+            else
+              @option_errors << column.formatted_errors
+            end
+          end
+        else
+          @option_errors << "The columns array for '#{@name}' is not uniform, it should contain arrays of two strings only"
+        end
+      else
+        @option_errors << "DbObject options for '#{@name}' must include a 'columns' array"
+      end
+
+      @index_column_options = @options["index_columns"]
+      @index_columns = []
+      if @index_column_options && @index_column_options.is_a?(Array)
+        @index_column_options.each do |option|
+          if option.is_a?(String) && temp_column_names.member?(option.to_sym)
+            @index_columns << option.to_sym
+          else
+            @option_errors << "The index_columns element: '#{option}' must be a column defined in the columns array"
+          end
+        end
+      end
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/loader.rb

@@ -0,0 +1,119 @@
+# encoding: utf-8
+require_relative "validatable"
+require_relative "db_object"
+require_relative "read_only_database"
+require "logstash/util/loggable"
+
+module LogStash module Filters module Jdbc
+  class Loader < Validatable
+    include LogStash::Util::Loggable
+
+    CONNECTION_ERROR_MSG = "Remote DB connection error when executing loader Jdbc query"
+
+    attr_reader :id, :table, :query, :max_rows
+    attr_reader :connection_string, :driver_library, :driver_class
+    attr_reader :user, :password, :staging_directory
+
+    def build_remote_db
+      @remote = ReadOnlyDatabase.create(connection_string, driver_class, driver_library, user, password)
+    end
+
+    def fetch
+      @remote.connect(CONNECTION_ERROR_MSG)
+      row_count = @remote.count(query)
+      if row_count.zero?
+        logger.warn? && logger.warn("Query returned no results", :lookup_id => @id, :query => query)
+        return @remote.empty_record_set
+      end
+      if row_count > max_rows
+        logger.warn? && logger.warn("Query returned more than max_rows results", :lookup_id => @id, :query => query, :count => row_count, :max_rows => max_rows)
+        return @remote.empty_record_set
+      end
+      @remote.query(query)
+    ensure
+      @remote.disconnect(CONNECTION_ERROR_MSG)
+    end
+
+    def close
+      @remote.disconnect(CONNECTION_ERROR_MSG)
+    end
+
+    private
+
+    def pre_initialize(options)
+      @table = options["local_table"]
+    end
+
+    def post_initialize
+      if valid?
+        @table = @table.to_sym
+      end
+    end
+
+    def parse_options
+      unless @table && @table.is_a?(String)
+        @option_errors << "The options must include a 'local_table' string"
+      end
+
+      @id = @options.fetch("id", @table)
+
+      @query = @options["query"]
+      unless @query && @query.is_a?(String)
+        @option_errors << "The options for '#{@table}' must include a 'query' string"
+      end
+
+      @max_rows = @options["max_rows"]
+      if @max_rows
+        if !@max_rows.respond_to?(:to_i)
+          @option_errors << "The 'max_rows' option for '#{@table}' must be an integer"
+        else
+          @max_rows = @max_rows.to_i
+        end
+      else
+        @max_rows = 1_000_000
+      end
+
+      @driver_library = @options["jdbc_driver_library"]
+      if @driver_library
+        if !@driver_library.is_a?(String)
+          @option_errors << "The 'jdbc_driver_library' option for '#{@table}' must be a string"
+        end
+        if !::File.exists?(@driver_library)
+          @option_errors << "The 'jdbc_driver_library' option for '#{@table}' must be a file that can be opened: #{driver_library}"
+        end
+      end
+
+      @driver_class = @options["jdbc_driver_class"]
+      if @driver_class && !@driver_class.is_a?(String)
+        @option_errors << "The 'jdbc_driver_class' option for '#{@table}' must be a string"
+      end
+
+      @connection_string = @options["jdbc_connection_string"]
+      if @connection_string && !@connection_string.is_a?(String)
+        @option_errors << "The 'jdbc_connection_string' option for '#{@table}' must be a string"
+      end
+
+      @user = @options["jdbc_user"]
+      if @user && !@user.is_a?(String)
+        @option_errors << "The 'jdbc_user' option for '#{@table}' must be a string"
+      end
+
+      @password = @options["jdbc_password"]
+      case @password
+      when String
+        @password = LogStash::Util::Password.new(@password)
+      when LogStash::Util::Password, nil
+        # this is OK
+      else
+        @option_errors << "The 'jdbc_password' option for '#{@table}' must be a string"
+      end
+
+      @staging_directory = @options["staging_directory"]
+      if @staging_directory
+        FileUtils.mkdir_p(@staging_directory)
+      end
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/loader_schedule.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+require_relative "validatable"
+require "rufus/scheduler"
+
+module LogStash module Filters module Jdbc
+  class LoaderSchedule < Validatable
+    attr_reader :schedule_frequency, :loader_schedule
+
+    def to_log_string
+      message = ""
+      message.concat "these months in the year [#{@cronline.months.to_a.join(", ")}];" unless @cronline.months.nil?
+      message.concat "these days in the month [#{@cronline.days.to_a.join(", ")}];" unless @cronline.days.nil?
+      message.concat "these hours in the day [#{@cronline.hours.to_a.join(", ")}];" unless @cronline.hours.nil?
+      message.concat "these minutes in the hour [#{@cronline.minutes.to_a.join(", ")}];" unless @cronline.minutes.nil?
+      message.concat "these seconds in the minute [#{@cronline.seconds.to_a.join(", ")}]" unless @cronline.seconds.nil?
+      if !message.empty?
+        message.prepend "Scheduled for: "
+      end
+      message
+    end
+
+    private
+
+    def post_initialize
+      if valid?
+        # From the Rufus::Scheduler docs:
+        #   By default, rufus-scheduler sleeps 0.300 second between every step.
+        #   At each step it checks for jobs to trigger and so on.
+        # set the frequency to 2.5 seconds if we are not reloading in the seconds timeframe
+        # rufus scheduler thread should respond to stop quickly enough.
+        if only_seconds_set?
+          @schedule_frequency = 0.3
+        else
+          @schedule_frequency = 2.5
+        end
+      end
+    end
+
+
+    def only_seconds_set?
+      @cronline.seconds &&
+        @cronline.minutes.nil? &&
+        @cronline.hours.nil? &&
+        @cronline.days.nil? &&
+        @cronline.months.nil?
+    end
+
+    def parse_options
+      @loader_schedule = @options
+
+      unless @loader_schedule.is_a?(String)
+        @option_errors << "The loader_schedule option must be a string"
+      end
+
+      begin
+        @cronline = Rufus::Scheduler::CronLine.new(@loader_schedule)
+      rescue => e
+        @option_errors << "The loader_schedule option is invalid: #{e.message}"
+      end
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/lookup.rb

@@ -0,0 +1,253 @@
+# encoding: utf-8
+require_relative "lookup_result"
+require "logstash/util/loggable"
+
+module LogStash module Filters module Jdbc
+  class Lookup
+    include LogStash::Util::Loggable
+
+    class Sprintfier
+      def initialize(param)
+        @param = param
+      end
+
+      def fetch(event, result)
+        formatted = event.sprintf(@param)
+        if formatted == @param # no field found so no transformation
+          result.invalid_parameters_push(@param)
+        end
+        formatted
+      end
+    end
+
+    class Getfier
+      def initialize(param)
+        @param = param
+      end
+
+      def fetch(event, result)
+        value = event.get(@param)
+        if value.nil? || value.is_a?(Hash) || value.is_a?(Array) # Array or Hash is not suitable
+          result.invalid_parameters_push(@param)
+        end
+        value
+      end
+    end
+
+    def self.find_validation_errors(array_of_options)
+      if !array_of_options.is_a?(Array)
+        return "The options must be an Array"
+        end
+      errors = []
+      array_of_options.each_with_index do |options, i|
+        instance = new(options, {}, "lookup-#{i.next}")
+        unless instance.valid?
+          errors << instance.formatted_errors
+        end
+      end
+      return nil if errors.empty?
+      errors.join("; ")
+    end
+
+    attr_reader :id, :target, :query, :parameters
+
+    def initialize(options, globals, default_id)
+      @id = options["id"] || default_id
+      @target = options["target"]
+      @id_used_as_target = @target.nil?
+      if @id_used_as_target
+        @target = @id
+      end
+      @options = options
+      @globals = globals
+      @valid = false
+      @option_errors = []
+      @default_result = nil
+      @prepared_statement = nil
+      @symbol_parameters = nil
+      parse_options
+    end
+
+    def id_used_as_target?
+      @id_used_as_target
+    end
+
+    def valid?
+      @valid
+    end
+
+    def formatted_errors
+      @option_errors.join(", ")
+    end
+
+    def enhance(local, event)
+      if @prepared_statement
+        result = call_prepared(local, event)
+      else
+        result = fetch(local, event) # should return a LookupResult
+      end
+      if result.failed? || result.parameters_invalid?
+        tag_failure(event)
+      end
+
+      if result.valid?
+        if @use_default && result.empty?
+          tag_default(event)
+          process_event(event, @default_result)
+        else
+          process_event(event, result)
+        end
+        true
+      else
+        false
+      end
+    end
+
+    def use_prepared_statement?
+      @prepared_parameters && !@prepared_parameters.empty?
+    end
+
+    def prepare(local)
+      hash = {}
+      @prepared_parameters.each_with_index { |v, i| hash[:"$p#{i}"] = v }
+      @prepared_param_placeholder_map = hash
+      @prepared_statement = local.prepare(query, hash.keys)
+    end
+
+    private
+
+    def tag_failure(event)
+      @tag_on_failure.each do |tag|
+        event.tag(tag)
+      end
+    end
+
+    def tag_default(event)
+      @tag_on_default_use.each do |tag|
+        event.tag(tag)
+      end
+    end
+
+    def fetch(local, event)
+      result = LookupResult.new()
+      if @parameters_specified
+        params = prepare_parameters_from_event(event, result)
+        if result.parameters_invalid?
+          logger.warn? && logger.warn("Parameter field not found in event", :lookup_id => @id, :invalid_parameters => result.invalid_parameters)
+          return result
+        end
+      else
+        params = {}
+      end
+      begin
+        logger.debug? && logger.debug("Executing Jdbc query", :lookup_id => @id, :statement => query, :parameters => params)
+        local.fetch(query, params).each do |row|
+          stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
+          result.push(stringified)
+        end
+      rescue ::Sequel::Error => e
+        # all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
+        result.failed!
+        logger.warn? && logger.warn("Exception when executing Jdbc query", :lookup_id => @id, :exception => e.message, :backtrace => e.backtrace.take(8))
+      end
+      # if either of: no records or a Sequel exception occurs the payload is
+      # empty and the default can be substituted later.
+      result
+    end
+
+    def call_prepared(local, event)
+      result = LookupResult.new()
+      if @parameters_specified
+        params = prepare_parameters_from_event(event, result)
+        if result.parameters_invalid?
+          logger.warn? && logger.warn("Parameter field not found in event", :lookup_id => @id, :invalid_parameters => result.invalid_parameters)
+          return result
+        end
+      else
+        params = {}
+      end
+      begin
+        logger.debug? && logger.debug("Executing Jdbc query", :lookup_id => @id, :statement => query, :parameters => params)
+        @prepared_statement.call(params).each do |row|
+          stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
+          result.push(stringified)
+        end
+      rescue ::Sequel::Error => e
+        # all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
+        result.failed!
+        logger.warn? && logger.warn("Exception when executing Jdbc query", :lookup_id => @id, :exception => e.message, :backtrace => e.backtrace.take(8))
+      end
+      # if either of: no records or a Sequel exception occurs the payload is
+      # empty and the default can be substituted later.
+      result
+    end
+
+    def process_event(event, result)
+      # use deep clone here so other filter function don't taint the payload by reference
+      event.set(@target, ::LogStash::Util.deep_clone(result.payload))
+    end
+
+    def prepare_parameters_from_event(event, result)
+      @symbol_parameters.inject({}) do |hash,(k,v)|
+        value = v.fetch(event, result)
+        hash[k] = value.is_a?(::LogStash::Timestamp) ? value.time : value
+        hash
+      end
+    end
+
+    def sprintf_or_get(v)
+      v.match(/%{([^}]+)}/) ? Sprintfier.new(v) : Getfier.new(v)
+    end
+
+    def parse_options
+      @query = @options["query"]
+      unless @query && @query.is_a?(String)
+        @option_errors << "The options for '#{@id}' must include a 'query' string"
+      end
+
+      if @options["parameters"] && @options["prepared_parameters"]
+        @option_errors << "Can't specify 'parameters' and 'prepared_parameters' in the same lookup"
+      else
+        @parameters = @options["parameters"]
+        @prepared_parameters = @options["prepared_parameters"]
+        @parameters_specified = false
+        if @parameters
+          if !@parameters.is_a?(Hash)
+            @option_errors << "The 'parameters' option for '#{@id}' must be a Hash"
+          else
+            # this is done once per lookup at start, i.e. Sprintfier.new et.al is done once.
+            @symbol_parameters = @parameters.inject({}) {|hash,(k,v)| hash[k.to_sym] = sprintf_or_get(v) ; hash }
+            # the user might specify an empty hash parameters => {}
+            # maybe due to an unparameterised query
+            @parameters_specified = !@symbol_parameters.empty?
+          end
+        elsif @prepared_parameters
+          if !@prepared_parameters.is_a?(Array)
+            @option_errors << "The 'prepared_parameters' option for '#{@id}' must be an Array"
+          elsif @query.count("?") != @prepared_parameters.size
+            @option_errors << "The 'prepared_parameters' option for '#{@id}' doesn't match count with query's placeholder"
+          else
+            #prepare the map @symbol_parameters :n => sprintf_or_get
+            hash = {}
+            @prepared_parameters.each_with_index {|v,i| hash[:"p#{i}"] = sprintf_or_get(v)}
+            @symbol_parameters = hash
+            @parameters_specified = !@prepared_parameters.empty?
+          end
+        end
+      end
+
+      default_hash = @options["default_hash"]
+      if default_hash && !default_hash.empty?
+        @default_result = LookupResult.new()
+        @default_result.push(default_hash)
+      end
+
+      @use_default = !@default_result.nil?
+
+      @tag_on_failure = @options["tag_on_failure"] || @globals["tag_on_failure"] || []
+      @tag_on_default_use = @options["tag_on_default_use"] || @globals["tag_on_default_use"] || []
+
+      @valid = @option_errors.empty?
+    end
+  end
+end end end

data/lib/logstash/filters/jdbc/lookup_processor.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+require_relative "lookup"
+require_relative "read_write_database"
+
+module LogStash module Filters module Jdbc
+  class LookupProcessor
+    attr_reader :lookups, :local
+
+    CONNECTION_ERROR_MSG = "Connection error when initialising lookup (local) db"
+    DISCONNECTION_ERROR_MSG = "Connection error when disconnecting from lookup (local) db"
+
+    def self.find_validation_errors(array_of_options)
+      if !array_of_options.is_a?(Array)
+        return "The options must be an Array"
+      end
+      errors = []
+      instance = new(array_of_options, {})
+      instance.lookups.each do |lookup|
+        unless lookup.valid?
+          errors << lookup.formatted_errors
+        end
+      end
+      unless instance.valid?
+        errors << instance.formatted_errors
+      end
+      return nil if errors.empty?
+      errors.join("; ")
+    end
+
+    def initialize(lookups_array, globals)
+      @lookups = lookups_array.map.with_index do |options, i|
+        Lookup.new(options, globals, "lookup-#{i.next}")
+      end
+      @lookups_errors = validate_lookups
+      if @lookups_errors.empty? && !globals.empty?
+        @local = ReadWriteDatabase.create(*globals.values_at(
+          "lookup_jdbc_connection_string",
+          "lookup_jdbc_driver_class",
+          "lookup_jdbc_driver_library").compact)
+        @local.connect(CONNECTION_ERROR_MSG)
+
+        create_prepared_statements_for_lookups
+      end
+    end
+
+    def enhance(event)
+      @lookups.map { |lookup| lookup.enhance(@local, event) }
+    end
+
+    def close
+      @local.disconnect(DISCONNECTION_ERROR_MSG)
+      @local = nil
+    end
+
+    def formatted_errors
+      @lookups_errors.join(", ")
+    end
+
+    def valid?
+      @lookups_errors.empty?
+    end
+
+    private
+
+    def create_prepared_statements_for_lookups()
+      @lookups.each do |lookup|
+        if lookup.use_prepared_statement?
+          lookup.prepare(@local)
+        end
+      end
+    end
+
+    def validate_lookups(lookups_errors = [])
+      ids = Hash.new(0)
+      errors = []
+      @lookups.each {|lookup| ids[lookup.id] += 1}
+      ids.select{|id, count| count > 1}.each do |id, count|
+        errors << "'#{id}' is specified multiple times"
+      end
+      if !errors.empty?
+        errors.unshift("Id setting must be different across all lookups")
+      end
+      lookups_errors.concat(errors)
+      targets = Hash.new {|h,k| h[k] = []}
+      errors = []
+      @lookups.each do |lookup|
+        # if id was used as target, skip target unique check because id uniqueness is checked already
+        next if lookup.id_used_as_target?
+        targets[lookup.target] << lookup.id
+      end
+      targets.select{|_,val| val.size > 1}.each do |target, ids|
+        errors << "'#{ids.join("', '")}' have the same target field setting"
+      end
+      if !errors.empty?
+        errors.unshift("Target setting must be different across all lookups")
+      end
+      lookups_errors.concat(errors)
+    end
+  end
+end end end