logstash-filter-jdbc_static 1.0.1 → 1.0.2

data/lib/logstash-filter-jdbc_static_jars.rb

@@ -1,5 +1,5 @@
-# encoding: utf-8
-require 'logstash/environment'
+# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
-root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")
+require 'jar_dependencies'
+require_jar('org.apache.derby', 'derby', '10.14.1.0')
+require_jar('org.apache.derby', 'derbyclient', '10.14.1.0')

data/lib/logstash/filters/jdbc/basic_database.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+require "fileutils"
 require "sequel"
 require "sequel/adapters/jdbc"
 require "java"
@@ -49,6 +51,8 @@ module LogStash module Filters module Jdbc
       rescue *CONNECTION_ERRORS => err
         # we do not raise an error when there is a connection error, we hope that the connection works next time
         logger.error(err_message, :exception => err.message, :backtrace => err.backtrace.take(8))
+      else
+        raise "::Sequel.connect returned a nil db instance, connection_string: #{@connection_string}, options: #{@options_hash.inspect}" if @db.nil?
       end
     end
 
@@ -80,7 +84,12 @@ module LogStash module Filters module Jdbc
 
     def verify_connection(connection_string, driver_class, driver_library, user, password)
       begin
-        require driver_library if driver_library
+        if driver_library
+          class_loader = java.lang.ClassLoader.getSystemClassLoader().to_java(java.net.URLClassLoader)
+          driver_library.split(",").each do |driver_path|
+            make_driver_path_loadable(class_loader, driver_path.strip)
+          end
+        end
       rescue LoadError => e
         msg = "The driver library cannot be loaded. The system error was: '#{e.message}'."
         raise wrap_error(ConnectionJdbcException, e, msg)
@@ -106,6 +115,11 @@ module LogStash module Filters module Jdbc
       end
     end
 
+    def make_driver_path_loadable(class_loader, driver_path)
+      # so we can set an expectation in rspec
+      class_loader.add_url(java.io.File.new(driver_path).toURI().toURL())
+    end
+
     def post_initialize()
       @unique_db_name = SecureRandom.hex(12)
     end

data/lib/logstash/filters/jdbc/column.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require_relative "validatable"
 
 module LogStash module Filters module Jdbc

data/lib/logstash/filters/jdbc/db_object.rb

@@ -1,10 +1,9 @@
+# encoding: utf-8
 require_relative "validatable"
 require_relative "column"
 
 module LogStash module Filters module Jdbc
 
-  TEMP_TABLE_PREFIX = "temp_".freeze
-
   class DbObject < Validatable
     #   {name => "servers", index_columns => ["ip"], columns => [["ip", "text"], ["name", "text"], ["location", "text"]]},
 
@@ -12,6 +11,9 @@ module LogStash module Filters module Jdbc
 
     def build(db)
       return unless valid?
+      if db.nil?
+        raise "DbObject given a database instance that is nil"
+      end
       schema_gen = db.create_table_generator()
       @columns.each {|col| schema_gen.column(col.name, col.datatype)}
       schema_gen.index(@index_columns)
@@ -27,10 +29,6 @@ module LogStash module Filters module Jdbc
       @name <=> other.name
     end
 
-    def as_temp_table_opts
-      {"name" => "#{TEMP_TABLE_PREFIX}#{@name}", "preserve_existing" => @preserve_existing, "index_columns" => @index_columns.map(&:to_s), "columns" => @columns.map(&:to_array)}
-    end
-
     def to_s
       inspect
     end

data/lib/logstash/filters/jdbc/loader.rb

@@ -10,9 +10,9 @@ module LogStash module Filters module Jdbc
 
     CONNECTION_ERROR_MSG = "Remote DB connection error when executing loader Jdbc query"
 
-    attr_reader :id, :table, :temp_table, :query, :max_rows
+    attr_reader :id, :table, :query, :max_rows
     attr_reader :connection_string, :driver_library, :driver_class
-    attr_reader :user, :password
+    attr_reader :user, :password, :staging_directory
 
     def build_remote_db
       @remote = ReadOnlyDatabase.create(connection_string, driver_class, driver_library, user, password)
@@ -46,7 +46,6 @@ module LogStash module Filters module Jdbc
 
     def post_initialize
       if valid?
-        @temp_table = "#{TEMP_TABLE_PREFIX}#{@table}".to_sym
         @table = @table.to_sym
       end
     end
@@ -108,6 +107,12 @@ module LogStash module Filters module Jdbc
       else
         @option_errors << "The 'jdbc_password' option for '#{@table}' must be a string"
       end
+
+      @staging_directory = @options["staging_directory"]
+      if @staging_directory
+        FileUtils.mkdir_p(@staging_directory)
+      end
+
       @valid = @option_errors.empty?
     end
   end

data/lib/logstash/filters/jdbc/loader_schedule.rb

@@ -1,24 +1,50 @@
+# encoding: utf-8
 require_relative "validatable"
+require "rufus/scheduler"
 
 module LogStash module Filters module Jdbc
   class LoaderSchedule < Validatable
     attr_reader :schedule_frequency, :loader_schedule
 
+    def to_log_string
+      message = ""
+      message.concat "these months in the year [#{@cronline.months.to_a.join(", ")}];" unless @cronline.months.nil?
+      message.concat "these days in the month [#{@cronline.days.to_a.join(", ")}];" unless @cronline.days.nil?
+      message.concat "these hours in the day [#{@cronline.hours.to_a.join(", ")}];" unless @cronline.hours.nil?
+      message.concat "these minutes in the hour [#{@cronline.minutes.to_a.join(", ")}];" unless @cronline.minutes.nil?
+      message.concat "these seconds in the minute [#{@cronline.seconds.to_a.join(", ")}]" unless @cronline.seconds.nil?
+      if !message.empty?
+        message.prepend "Scheduled for: "
+      end
+      message
+    end
+
     private
 
     def post_initialize
       if valid?
         # From the Rufus::Scheduler docs:
-        # By default, rufus-scheduler sleeps 0.300 second between every step.
-        # At each step it checks for jobs to trigger and so on.
-        if @cronline.seconds.is_a?(Set)
+        #   By default, rufus-scheduler sleeps 0.300 second between every step.
+        #   At each step it checks for jobs to trigger and so on.
+        # set the frequency to 2.5 seconds if we are not reloading in the seconds timeframe
+        # rufus scheduler thread should respond to stop quickly enough.
+        if only_seconds_set?
           @schedule_frequency = 0.3
         else
-          @schedule_frequency = 30
+          @schedule_frequency = 2.5
         end
       end
     end
 
+
+    def only_seconds_set?
+      @cronline.seconds &&
+        @cronline.minutes.nil? &&
+        @cronline.hours.nil? &&
+        @cronline.days.nil? &&
+        @cronline.months.nil?
+    end
+
     def parse_options
       @loader_schedule = @options
 

data/lib/logstash/filters/jdbc/lookup.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require_relative "lookup_result"
 require "logstash/util/loggable"
 

data/lib/logstash/filters/jdbc/lookup_processor.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require_relative "lookup"
 require_relative "read_write_database"
 
@@ -23,16 +24,14 @@ module LogStash module Filters module Jdbc
         errors << instance.formatted_errors
       end
       return nil if errors.empty?
-      # errors.unshift("For plugin #{}")
       errors.join("; ")
     end
 
     def initialize(lookups_array, globals)
-      @lookups_errors = []
       @lookups = lookups_array.map.with_index do |options, i|
         Lookup.new(options, globals, "lookup-#{i.next}")
       end
-      validate_lookups
+      @lookups_errors = validate_lookups
       if @lookups_errors.empty? && !globals.empty?
         @local = ReadWriteDatabase.create(*globals.values_at(
           "lookup_jdbc_connection_string",
@@ -61,7 +60,7 @@ module LogStash module Filters module Jdbc
 
     private
 
-    def validate_lookups
+    def validate_lookups(lookups_errors = [])
       ids = Hash.new(0)
       errors = []
       @lookups.each {|lookup| ids[lookup.id] += 1}
@@ -71,7 +70,7 @@ module LogStash module Filters module Jdbc
       if !errors.empty?
         errors.unshift("Id setting must be different across all lookups")
       end
-      @lookups_errors.concat(errors)
+      lookups_errors.concat(errors)
       targets = Hash.new {|h,k| h[k] = []}
       errors = []
       @lookups.each do |lookup|
@@ -85,7 +84,7 @@ module LogStash module Filters module Jdbc
       if !errors.empty?
         errors.unshift("Target setting must be different across all lookups")
       end
-      @lookups_errors.concat(errors)
+      lookups_errors.concat(errors)
     end
   end
 end end end

data/lib/logstash/filters/jdbc/lookup_result.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 module LogStash module Filters module Jdbc
   class LookupResult
     attr_reader :payload, :invalid_parameters

data/lib/logstash/filters/jdbc/read_write_database.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require_relative "basic_database"
 
 module LogStash module Filters module Jdbc
@@ -5,13 +6,13 @@ module LogStash module Filters module Jdbc
     def repopulate_all(loaders)
       case loaders.size
         when 1
-          fill_and_switch(loaders.first)
+          fill_local_table(loaders.first)
         when 2
-          fill_and_switch(loaders.first)
-          fill_and_switch(loaders.last)
+          fill_local_table(loaders.first)
+          fill_local_table(loaders.last)
         else
           loaders.each do |loader|
-            fill_and_switch(loader)
+            fill_local_table(loader)
           end
       end
     end
@@ -50,26 +51,37 @@ module LogStash module Filters module Jdbc
 
     private
 
-    def fill_and_switch(loader)
+    def fill_local_table(loader)
       begin
-        records = loader.fetch
-        return if records.size.zero?
         @rwlock.writeLock().lock()
-        tmp = self.class.random_name
-        @db.transaction do |conn|
-          @db[loader.temp_table].multi_insert(records)
-          @db.rename_table(loader.temp_table, tmp)
-          @db.rename_table(loader.table, loader.temp_table)
-          @db.rename_table(tmp, loader.table)
-          @db[loader.temp_table].truncate
+        start = Time.now.to_f
+        records = loader.fetch
+        records_size = records.size
+        return if records_size.zero?
+        logger.info("loader #{loader.id}, fetched #{records_size} records in: #{(Time.now.to_f - start).round(3)} seconds")
+        start = Time.now.to_f
+        import_file = ::File.join(loader.staging_directory, loader.table.to_s)
+        ::File.open(import_file, "w") do |fd|
+          dataset = @db[loader.table]
+          records.each do |hash|
+            array = hash.values.map {|val| dataset.literal(val) }
+            fd.puts(array.join(", "))
+          end
+          fd.fsync
         end
+        logger.info("loader #{loader.id}, saved fetched records to import file in: #{(Time.now.to_f - start).round(3)} seconds")
+        start = Time.now.to_f
+        import_cmd = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE (null,'#{loader.table.upcase}','#{import_file}',null,'''',null,1)"
+        @db.execute_ddl(import_cmd)
+        FileUtils.rm_f(import_file)
+        logger.info("loader #{loader.id}, imported all fetched records in: #{(Time.now.to_f - start).round(3)} seconds")
       rescue *CONNECTION_ERRORS => err
         # we do not raise an error when there is a connection error, we hope that the connection works next time
-        logger.error("Connection error when filling lookup db from loader query results", :exception => err.message, :backtrace => err.backtrace.take(8))
+        logger.error("Connection error when filling lookup db from loader #{loader.id}, query results", :exception => err.message, :backtrace => err.backtrace.take(8))
       rescue => err
         # In theory all exceptions in Sequel should be wrapped in Sequel::Error
         # There are cases where exceptions occur in unprotected ensure sections
-        msg = "Exception when filling lookup db from loader query results, original exception: #{err.class}, original message: #{err.message}"
+        msg = "Exception when filling lookup db from loader #{loader.id}, query results, original exception: #{err.class}, original message: #{err.message}"
         logger.error(msg, :backtrace => err.backtrace.take(16))
         raise wrap_error(LoaderJdbcException, err, msg)
       ensure

data/lib/logstash/filters/jdbc/repeating_load_runner.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require_relative "single_load_runner"
 
 module LogStash module Filters module Jdbc
@@ -6,6 +7,7 @@ module LogStash module Filters module Jdbc
 
     def repeated_load
       local.repopulate_all(loaders)
+      @reload_counter.increment
     end
   end
 end end end

data/lib/logstash/filters/jdbc/single_load_runner.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require_relative 'db_object'
 
 module LogStash module Filters module Jdbc
@@ -9,12 +10,9 @@ module LogStash module Filters module Jdbc
       @local = local
       @loaders = loaders
       @preloaders = []
+      @reload_counter = Concurrent::AtomicFixnum.new(0)
       preloaders.map do |pre|
-        dbo = DbObject.new(pre)
-        @preloaders << dbo
-        hash = dbo.as_temp_table_opts
-        _dbo = DbObject.new(hash)
-        @preloaders << _dbo if _dbo.valid?
+        @preloaders << DbObject.new(pre)
       end
       @preloaders.sort!
     end
@@ -22,6 +20,7 @@ module LogStash module Filters module Jdbc
     def initial_load
       do_preload
       local.populate_all(loaders)
+      @reload_counter.increment
     end
 
     def repeated_load
@@ -31,6 +30,10 @@ module LogStash module Filters module Jdbc
       repeated_load
     end
 
+    def reload_count
+      @reload_counter.value
+    end
+
     private
 
     def do_preload

data/lib/logstash/filters/jdbc/validatable.rb

@@ -1,11 +1,8 @@
 # encoding: utf-8
-
 module LogStash module Filters module Jdbc
   class Validatable
-    def self.find_validation_errors(array_of_options)
-      if !array_of_options.is_a?(Array)
-        return "The options must be an Array"
-      end
+    def self.find_validation_errors(options)
+      array_of_options = Array(options)
       errors = []
       array_of_options.each do |options|
         instance = new(options)

data/lib/logstash/filters/jdbc_static.rb

@@ -77,7 +77,7 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
   #
   # There is no schedule by default. If no schedule is given, then the loaders are run
   # exactly once.
-  config :loader_schedule, :validate => [LogStash::Filters::Jdbc::LoaderSchedule]
+  config :loader_schedule, :required => false, :validate => LogStash::Filters::Jdbc::LoaderSchedule
 
   # Append values to the `tags` field if sql error occured
   # Alternatively, individual `tag_on_failure` arrays can be added to each lookup hash
@@ -87,7 +87,8 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
   config :tag_on_default_use, :validate => :array, :default => ["_jdbcstaticdefaultsused"]
 
   # Remote Load DB Jdbc driver library path to third party driver library.
-  config :jdbc_driver_library, :validate => :path
+  # Use comma separated paths in one string if you need more than one library.
+  config :jdbc_driver_library, :validate => :string
 
   # Remote Load DB Jdbc driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver"
   config :jdbc_driver_class, :validate => :string, :required => true
@@ -101,6 +102,9 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
   # Remote Load DB Jdbc password
   config :jdbc_password, :validate => :password
 
+  # directory for temp files created during bulk loader import.
+  config :staging_directory, :validate => :string, :default => ::File.join(Dir.tmpdir, "logstash", config_name, "import_data")
+
   # NOTE: For the initial release, we are not allowing the user to specify their own local lookup JDBC DB settings.
   # In the near future we have to consider identical config running in multiple pipelines stomping over each other
   # when the database names are common across configs because there is only one Derby server in memory per JVM.
@@ -123,6 +127,11 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
         unless validation_errors.nil?
           return false, validation_errors
         end
+      elsif validator.respond_to?(:find_validation_errors)
+        validation_errors = validator.find_validation_errors(value)
+        unless validation_errors.nil?
+          return false, validation_errors
+        end
       else
         return old_validate_value(value, validator)
       end
@@ -135,7 +144,6 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
   def register
     prepare_data_dir
     prepare_runner
-    @loader_runner.initial_load
   end
 
   def filter(event)
@@ -149,12 +157,18 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
     @processor.close
   end
 
+  def loader_runner
+    # use for test verification
+    @loader_runner
+  end
+
   private
 
   def prepare_data_dir
     # later, when local persistent databases are allowed set this property to LS_HOME/data/jdbc-static/
     # must take multi-pipelines into account and more than one config using the same jdbc-static settings
     java.lang.System.setProperty("derby.system.home", ENV["HOME"])
+    logger.info("derby.system.home is: #{java.lang.System.getProperty("derby.system.home")}")
   end
 
   def prepare_runner
@@ -168,18 +182,20 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
     @processor = Jdbc::LookupProcessor.new(@local_lookups, global_lookup_options)
     runner_args.unshift(@processor.local)
     if @loader_schedule
-      require "rufus/scheduler"
       args = []
       @loader_runner = Jdbc::RepeatingLoadRunner.new(*runner_args)
-
+      @loader_runner.initial_load
       cronline = Jdbc::LoaderSchedule.new(@loader_schedule)
+      cronline.to_log_string.tap do |msg|
+        logger.info("Scheduler operations: #{msg}") unless msg.empty?
+      end
+      logger.info("Scheduler scan for work frequency is: #{cronline.schedule_frequency}")
       rufus_args = {:max_work_threads => 1, :frequency => cronline.schedule_frequency}
-
       @scheduler = Rufus::Scheduler.new(rufus_args)
       @scheduler.cron(cronline.loader_schedule, @loader_runner)
-      @scheduler.join
     else
       @loader_runner = Jdbc::SingleLoadRunner.new(*runner_args)
+      @loader_runner.initial_load
     end
   end
 
@@ -212,5 +228,8 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
     if @jdbc_password
       options["jdbc_password"] = @jdbc_password
     end
+    if @staging_directory
+      options["staging_directory"] = @staging_directory
+    end
   end
 end end end