logstash-integration-jdbc 5.0.0.alpha1

data/lib/logstash/plugin_mixins/jdbc_streaming/statement_handler.rb

@@ -0,0 +1,143 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+
+module LogStash module PluginMixins module JdbcStreaming
+  # so as to not clash with the class of the same name and function in the jdbc input
+  # this is in the `module JdbcStreaming` namespace
+  # this duplication can be removed in a universal plugin
+
+  class StatementHandler
+    def self.build_statement_handler(plugin)
+      klass = plugin.use_prepared_statements ? PreparedStatementHandler : NormalStatementHandler
+      klass.new(plugin)
+    end
+
+    attr_reader :statement, :parameters, :cache
+
+    def initialize(plugin)
+      @statement = plugin.statement
+      klass = plugin.use_cache ? RowCache : NoCache
+      @cache = klass.new(plugin.cache_size, plugin.cache_expiration)
+      post_init(plugin)
+    end
+
+    # Get from cache or performs remote lookup and saves to cache
+    # @param db [Sequel::Database]
+    # @param event [LogStash::Event]
+    # @returnparam [CachePayload]
+    def cache_lookup(db, event)
+      # override in subclass
+    end
+
+    private
+
+    def common_cache_lookup(db, event)
+      params = prepare_parameters_from_event(event)
+      @cache.get(params) do
+        result = CachePayload.new
+        begin
+          logger.debug? && logger.debug("Executing JDBC query", :statement => statement, :parameters => params)
+          execute_extract_records(db, params, result)
+        rescue ::Sequel::Error => e
+          # all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
+          result.failed!
+          logger.warn? && logger.warn("Exception when executing JDBC query", :statement => statement, :parameters => params, :exception => e)
+        end
+        # if either of: no records or a Sequel exception occurs the payload is
+        # empty and the default can be substituted later.
+        result
+      end
+    end
+
+    def execute_extract_records(db, params, result)
+      # override in subclass
+    end
+
+    def post_init(plugin)
+      # override in subclass, if needed
+    end
+
+    def prepare_parameters_from_event(event)
+      @parameters.inject({}) do |hash, (k, parameter_handler)|
+        # defer to appropriate parameter handler
+        value = parameter_handler.extract_from(event)
+        hash[k] = value.is_a?(::LogStash::Timestamp) ? value.time : value
+        hash
+      end
+    end
+  end
+
+  class NormalStatementHandler < StatementHandler
+    include LogStash::Util::Loggable
+
+    # Get from cache or performs remote lookup and saves to cache
+    # @param db [Sequel::Database]
+    # @param event [LogStash::Event]
+    # @returnparam [CachePayload]
+    def cache_lookup(db, event)
+      common_cache_lookup(db, event)
+    end
+
+    private
+
+    def execute_extract_records(db, params, result)
+      dataset = db[statement, params] # returns a Sequel dataset
+      dataset.all do |row|
+        result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} # Stringify row keys
+      end
+    end
+
+    def post_init(plugin)
+      @parameters = plugin.parameters
+    end
+  end
+
+  class PreparedStatementHandler < StatementHandler
+    include LogStash::Util::Loggable
+    attr_reader :name, :bind_values_array, :statement_prepared, :prepared
+
+    # Get from cache or performs remote lookup and saves to cache
+    # @param db [Sequel::Database]
+    # @param event [LogStash::Event]
+    # @returnparam [CachePayload]
+    def cache_lookup(db, event)
+      build_prepared_statement(db)
+      common_cache_lookup(db, event)
+    end
+
+    private
+
+    def execute_extract_records(db, params, result)
+      records = db.call(name, params) # returns an array of hashes
+      records.each do |row|
+        result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
+      end
+    end
+
+    def post_init(plugin)
+      @name = plugin.prepared_statement_name.to_sym
+      @bind_values_array = plugin.prepared_statement_bind_values
+      @statement_prepared = Concurrent::AtomicBoolean.new(false)
+      @parameters = create_bind_values_hash
+    end
+
+    def build_prepared_statement(db)
+      # create prepared statement on first use
+      if statement_prepared.false?
+        prepended = parameters.keys.map{|v| v.to_s.prepend("$").to_sym}
+        @prepared = db[statement, *prepended].prepare(:select, name)
+        statement_prepared.make_true
+      end
+      # make sure the Sequel database instance has the prepared statement
+      if db.prepared_statement(name).nil?
+        db.set_prepared_statement(name, prepared)
+      end
+    end
+
+    def create_bind_values_hash
+      hash = {}
+      bind_values_array.each_with_index {|v,i| hash[:"p#{i}"] = v}
+      hash
+    end
+  end
+end end end

data/lib/logstash/plugin_mixins/jdbc_streaming.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+require "logstash/config/mixin"
+
+# Tentative of abstracting JDBC logic to a mixin
+# for potential reuse in other plugins (input/output)
+module LogStash module PluginMixins module JdbcStreaming
+  class RowCache
+    def initialize(size, ttl)
+      @cache = ::LruRedux::TTL::ThreadSafeCache.new(size, ttl)
+    end
+
+    def get(parameters)
+      @cache.getset(parameters) { yield }
+    end
+  end
+
+  class NoCache
+    def initialize(size, ttl) end
+
+    def get(statement)
+      yield
+    end
+  end
+
+  # This method is called when someone includes this module
+  def self.included(base)
+    # Add these methods to the 'base' given.
+    base.extend(self)
+    base.setup_jdbc_config
+  end
+
+  public
+  def setup_jdbc_config
+    # JDBC driver library path to third party driver library.
+    config :jdbc_driver_library, :validate => :path
+
+    # JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver"
+    config :jdbc_driver_class, :validate => :string, :required => true
+
+    # JDBC connection string
+    config :jdbc_connection_string, :validate => :string, :required => true
+
+    # JDBC user
+    config :jdbc_user, :validate => :string
+
+    # JDBC password
+    config :jdbc_password, :validate => :password
+
+    # Connection pool configuration.
+    # Validate connection before use.
+    config :jdbc_validate_connection, :validate => :boolean, :default => false
+
+    # Connection pool configuration.
+    # How often to validate a connection (in seconds)
+    config :jdbc_validation_timeout, :validate => :number, :default => 3600
+  end
+
+  private
+
+  def load_driver_jars
+    unless @jdbc_driver_library.nil? || @jdbc_driver_library.empty?
+      @jdbc_driver_library.split(",").each do |driver_jar|
+        begin
+          @logger.debug("loading #{driver_jar}")
+          # Use https://github.com/jruby/jruby/wiki/CallingJavaFromJRuby#from-jar-files to make classes from jar
+          # available
+          require driver_jar
+        rescue LoadError => e
+          raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
+        end
+      end
+    end
+  end
+
+  public
+  def prepare_jdbc_connection
+    require "sequel"
+    require "sequel/adapters/jdbc"
+    require "java"
+
+    load_driver_jars
+
+    @sequel_opts_symbols = @sequel_opts.inject({}) {|hash, (k,v)| hash[k.to_sym] = v; hash}
+    @sequel_opts_symbols[:user] = @jdbc_user unless @jdbc_user.nil? || @jdbc_user.empty?
+    @sequel_opts_symbols[:password] = @jdbc_password.value unless @jdbc_password.nil?
+
+    Sequel::JDBC.load_driver(@jdbc_driver_class)
+    @database = Sequel.connect(@jdbc_connection_string, @sequel_opts_symbols)
+    if @jdbc_validate_connection
+      @database.extension(:connection_validator)
+      @database.pool.connection_validation_timeout = @jdbc_validation_timeout
+    end
+    begin
+      @database.test_connection
+    rescue Sequel::DatabaseConnectionError => e
+      #TODO return false and let the plugin raise a LogStash::ConfigurationError
+      raise e
+    end
+  end # def prepare_jdbc_connection
+end end end

data/lib/logstash-integration-jdbc_jars.rb

@@ -0,0 +1,5 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
+
+require 'jar_dependencies'
+require_jar('org.apache.derby', 'derby', '10.14.1.0')
+require_jar('org.apache.derby', 'derbyclient', '10.14.1.0')

data/logstash-integration-jdbc.gemspec

@@ -0,0 +1,44 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-integration-jdbc'
+  s.version         = '5.0.0.alpha1'
+  s.licenses = ['Apache License (2.0)']
+  s.summary         = "Integration with JDBC - input and filter plugins"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Elastic"]
+  s.email = 'info@elastic.co'
+  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths   = ["lib", "vendor/jar-dependencies"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = {
+      "logstash_plugin"     => "true",
+      "logstash_group"      => "integration",
+      "integration_plugins" => "logstash-input-jdbc,logstash-filter-jdbc_streaming,logstash-filter-jdbc_static"
+  }
+
+  # Gem dependencies
+  s.add_development_dependency 'jar-dependencies', '~> 0.3'
+
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  # Restrict use of this plugin to versions of Logstash where support for integration plugins is present.
+  s.add_runtime_dependency "logstash-core", ">= 6.5.0"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'sequel'
+  s.add_runtime_dependency 'lru_redux' # lru cache with ttl
+
+  s.add_runtime_dependency 'tzinfo'
+  s.add_runtime_dependency 'tzinfo-data'
+  # 3.5 limitation is required for jdbc-static loading schedule
+  s.add_runtime_dependency 'rufus-scheduler', '< 3.5'
+
+  s.add_development_dependency "childprocess"
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'timecop'
+  s.add_development_dependency 'jdbc-derby'
+end

data/spec/filters/env_helper.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+# use the rspec --require command line option to have this file evaluated before rspec runs
+# it i
+
+GEM_BASE_DIR = ::File.expand_path("../../..", __FILE__)
+BASE_DERBY_DIR = ::File.join(GEM_BASE_DIR, "spec", "helpers")
+ENV["HOME"] = GEM_BASE_DIR
+ENV["TEST_DEBUG"] = "true"
+java.lang.System.setProperty("ls.logs", "console")

data/spec/filters/integration/jdbc_static_spec.rb

@@ -0,0 +1,154 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc_static"
+require "sequel"
+require "sequel/adapters/jdbc"
+require "stud/temporary"
+require "timecop"
+
+module LogStash module Filters
+  describe JdbcStatic, :integration => true do
+
+    before(:all) do
+      @thread_abort = Thread.abort_on_exception
+      Thread.abort_on_exception = true
+    end
+
+    let(:loader_statement) { "SELECT ip, name, location FROM reference_table" }
+    let(:lookup_statement) { "SELECT * FROM servers WHERE ip LIKE :ip" }
+    let(:parameters_rhs) { "%%{[ip]}" }
+    let(:temp_import_path_plugin) { Stud::Temporary.pathname }
+    let(:temp_import_path_rspec) { Stud::Temporary.pathname }
+
+    ENV["TZ"] = "Etc/UTC"
+
+    # For Travis and CI based on docker, we source from ENV
+    jdbc_connection_string = ENV.fetch("PG_CONNECTION_STRING",
+                                       "jdbc:postgresql://postgresql:5432") + "/jdbc_static_db?user=postgres"
+
+    let(:local_db_objects) do
+      [
+        {"name" => "servers", "index_columns" => ["ip"], "columns" => [["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+      ]
+    end
+
+    let(:settings) do
+      {
+        "jdbc_user" => ENV['USER'],
+        "jdbc_driver_class" => "org.postgresql.Driver",
+        "jdbc_driver_library" => "/usr/share/logstash/postgresql.jar",
+        "staging_directory" => temp_import_path_plugin,
+        "jdbc_connection_string" => jdbc_connection_string,
+        "loaders" => [
+          {
+            "id" =>"servers",
+            "query" => loader_statement,
+            "local_table" => "servers"
+          }
+        ],
+        "local_db_objects" => local_db_objects,
+        "local_lookups" => [
+          {
+            "query" => lookup_statement,
+            "parameters" => {"ip" => parameters_rhs},
+            "target" => "server"
+          }
+        ]
+      }
+    end
+
+    let(:plugin) { JdbcStatic.new(settings) }
+
+    let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
+
+    let(:ipaddr) { ".3.1.1" }
+
+    describe "non scheduled operation" do
+      after { plugin.close }
+
+      context "under normal conditions" do
+        it "enhances an event" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
+        end
+      end
+
+      context "when the loader query returns no results" do
+        let(:loader_statement) { "SELECT ip, name, location FROM reference_table WHERE ip LIKE '20%'" }
+        it "add an empty array to the target field" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([])
+        end
+      end
+
+      context "under normal conditions with prepared statement" do
+        let(:lookup_statement) { "SELECT * FROM servers WHERE ip LIKE ?" }
+        let(:settings) do
+          {
+            "jdbc_user" => ENV['USER'],
+            "jdbc_driver_class" => "org.postgresql.Driver",
+            "jdbc_driver_library" => "/usr/share/logstash/postgresql.jar",
+            "staging_directory" => temp_import_path_plugin,
+            "jdbc_connection_string" => jdbc_connection_string,
+            "loaders" => [
+              {
+                "id" =>"servers",
+                "query" => loader_statement,
+                "local_table" => "servers"
+              }
+            ],
+            "local_db_objects" => local_db_objects,
+            "local_lookups" => [
+              {
+                "query" => lookup_statement,
+                "prepared_parameters" => [parameters_rhs],
+                "target" => "server"
+              }
+            ]
+          }
+        end
+
+        it "enhances an event" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
+        end
+      end
+
+      context "under normal conditions when index_columns is not specified" do
+        let(:local_db_objects) do
+          [
+            {"name" => "servers", "columns" => [["ip", "varchar(64)"], ["name", "varchar(64)"], ["location", "varchar(64)"]]},
+          ]
+        end
+        it "enhances an event" do
+          plugin.register
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
+        end
+      end
+    end
+
+    describe "scheduled operation" do
+      context "given a loader_schedule" do
+        it "should properly schedule" do
+          settings["loader_schedule"] = "*/10 * * * * * UTC"
+          Timecop.travel(Time.now.utc - 3600)
+          Timecop.scale(60)
+          static_filter = JdbcStatic.new(settings)
+          runner = Thread.new(static_filter) do |filter|
+            filter.register
+          end
+          sleep 3
+          static_filter.filter(event)
+          expect(static_filter.loader_runner.reload_count).to be > 1
+          static_filter.close
+          Timecop.return
+          expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
+        end
+      end
+    end
+  end
+end end

data/spec/filters/integration/jdbcstreaming_spec.rb

@@ -0,0 +1,173 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc_streaming"
+require "sequel"
+require "sequel/adapters/jdbc"
+
+module LogStash module Filters
+  class TestJdbcStreaming < JdbcStreaming
+    attr_reader :database
+  end
+
+  describe JdbcStreaming, :integration => true do
+    ENV["TZ"] = "Etc/UTC"
+
+    # For Travis and CI based on docker, we source from ENV
+    jdbc_connection_string = ENV.fetch("PG_CONNECTION_STRING",
+      "jdbc:postgresql://postgresql:5432") + "/jdbc_streaming_db?user=postgres"
+
+    let(:mixin_settings) do
+      { "jdbc_driver_class" => "org.postgresql.Driver",
+        "jdbc_driver_library" => "/usr/share/logstash/postgresql.jar",
+        "jdbc_connection_string" => jdbc_connection_string
+      }
+    end
+    let(:plugin) { JdbcStreaming.new(mixin_settings.merge(settings)) }
+    let(:db) do
+      ::Sequel.connect(mixin_settings['jdbc_connection_string'])
+    end
+    let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
+    let(:cache_expiration) { 3.0 }
+    let(:use_cache) { true }
+    let(:cache_size) { 10 }
+    let(:statement) { "SELECT name, location FROM reference_table WHERE ip = :ip" }
+    let(:settings) do
+      {
+        "statement" => statement,
+        "parameters" => {"ip" => "ip"},
+        "target" => "server",
+        "use_cache" => use_cache,
+        "cache_expiration" => cache_expiration,
+        "cache_size" => cache_size,
+        "tag_on_failure" => ["lookup_failed"],
+        "tag_on_default_use" => ["default_used_instead"],
+        "default_hash" => {"name" => "unknown", "location" => "unknown"},
+        "sequel_opts" => {"pool_timeout" => 600}
+      }
+    end
+    let(:ipaddr)    { "10.#{idx}.1.1" }
+
+    before :each do
+      plugin.register
+    end
+
+    describe "found record - uses row" do
+      let(:idx) { 200 }
+
+      it "fills in the target" do
+        plugin.filter(event)
+        expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+        expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+      end
+    end
+
+    describe "In Prepared Statement mode, found record - uses row" do
+      let(:idx) { 200 }
+      let(:statement) { "SELECT name, location FROM reference_table WHERE ip = ?" }
+      let(:settings) do
+        {
+          "statement" => statement,
+          "use_prepared_statements" => true,
+          "prepared_statement_name" => "lookup_ip",
+          "prepared_statement_bind_values" => ["[ip]"],
+          "target" => "server",
+          "use_cache" => use_cache,
+          "cache_expiration" => cache_expiration,
+          "cache_size" => cache_size,
+          "tag_on_failure" => ["lookup_failed"],
+          "tag_on_default_use" => ["default_used_instead"],
+          "default_hash" => {"name" => "unknown", "location" => "unknown"},
+          "sequel_opts" => {"pool_timeout" => 600}
+        }
+      end
+      it "fills in the target" do
+        plugin.filter(event)
+        expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+        expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+      end
+    end
+
+    context "when fetching from cache" do
+      let(:plugin) { TestJdbcStreaming.new(mixin_settings.merge(settings)) }
+      let(:events) do
+        5.times.map{|i| ::LogStash::Event.new("message" => "some other text #{i}", "ip" => ipaddr) }
+      end
+      let(:call_count) { 1 }
+      before(:each) do
+        expect(plugin.database).to receive(:[]).exactly(call_count).times.and_call_original
+        plugin.filter(event)
+      end
+
+      describe "found record - caches row" do
+        let(:idx)    { "42" }
+        it "calls the database once then uses the cache" do
+          expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+          expect(event.get("tags") || []).not_to include("lookup_failed")
+          expect(event.get("tags") || []).not_to include("default_used_instead")
+          events.each do |evt|
+            plugin.filter(evt)
+            expect(evt.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+          end
+        end
+      end
+
+      describe "missing record - uses default" do
+        let(:idx)    { "252" }
+        it "calls the database once then uses the cache" do
+          expect(event.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+          expect(event.get("tags") & ["lookup_failed", "default_used_instead"]).to eq(["default_used_instead"])
+          events.each do |evt|
+            plugin.filter(evt)
+            expect(evt.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+          end
+        end
+      end
+
+      context "extremely small cache expiration" do
+        describe "found record - cache always expires" do
+          let(:idx)    { "10" }
+          let(:call_count) { 6 }
+          let(:cache_expiration) { 0.0000001 }
+          it "calls the database each time because cache entry expired" do
+            expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+            expect(event.get("tags") || []).not_to include("lookup_failed")
+            expect(event.get("tags") || []).not_to include("default_used_instead")
+            events.each do |evt|
+              plugin.filter(evt)
+              expect(evt.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+            end
+          end
+        end
+      end
+
+      context "when cache is disabled" do
+        let(:call_count) { 6 }
+        let(:use_cache) { false }
+        describe "database is always called" do
+          let(:idx)    { "1" }
+          it "calls the database each time" do
+            expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+            expect(event.get("tags") || []).not_to include("lookup_failed")
+            expect(event.get("tags") || []).not_to include("default_used_instead")
+            events.each do |evt|
+              plugin.filter(evt)
+              expect(evt.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+            end
+          end
+        end
+
+        describe "database is always called but record is missing and default is used" do
+          let(:idx)    { "251" }
+          it "calls the database each time" do
+            expect(event.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+            expect(event.get("tags") & ["lookup_failed", "default_used_instead"]).to eq(["default_used_instead"])
+            events.each do |evt|
+              plugin.filter(evt)
+              expect(evt.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+            end
+          end
+        end
+      end
+    end
+  end
+
+end end