logstash-filter-jdbc_streaming 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0a1ec4d5c58f6ce550226691dca0007a13c4b136
+  data.tar.gz: a3a51d17ea4a55cf57e74537a783c651cd737286
+SHA512:
+  metadata.gz: 6ce0c39c40c7f0e93776bd9abdf185cc3780d96d3742e051488db497175fc3929a9bfe04943bde7e26b6f0d1a915363e3e00e28a3c77d6de6242ec2ddadc30e7
+  data.tar.gz: adf5c47acdac61a900d4f2288aa466bbcbdc28165ab7d6d28fc666eedb1e294024348127f789f69ec7f4c2b62d11c86491c627149d69d11f083672953f25b2b0

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.0
+ - Initial release
+ - Added LRU + TTL Cache

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,98 @@
+# Logstash Plugin
+
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-jdbc_streaming.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-jdbc_streaming)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
+
+- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run unit tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Install plugin
+```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/jdbc_streaming.rb

@@ -0,0 +1,196 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "logstash/plugin_mixins/jdbc_streaming"
+require "lru_redux"
+
+# This filter executes a SQL query and store the result set in the field
+# specified as `target`.
+# It will cache the results locally in an LRU cache with expiry
+#
+# For example you can load a row based on an id from in the event
+#
+# [source,ruby]
+# filter {
+#   jdbc_streaming {
+#     jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar"
+#     jdbc_driver_class => "com.mysql.jdbc.Driver"
+#     jdbc_connection_string => ""jdbc:mysql://localhost:3306/mydatabase"
+#     jdbc_user => "me"
+#     jdbc_password => "secret"
+#     statement => "select * from WORLD.COUNTRY WHERE Code = :code"
+#     parameters => { "code" => "country_code"}
+#     target => "country_details"
+#   }
+# }
+#
+module LogStash module Filters class JdbcStreaming < LogStash::Filters::Base
+  class CachePayload
+    attr_reader :payload
+    def initialize
+      @failure = false
+      @payload = []
+    end
+
+    def push(data)
+      @payload << data
+    end
+
+    def failed!
+      @failure = true
+    end
+
+    def failed?
+      @failure
+    end
+
+    def empty?
+      @payload.empty?
+    end
+  end
+
+  class RowCache
+    def initialize(size, ttl)
+      @cache = ::LruRedux::TTL::ThreadSafeCache.new(size, ttl)
+    end
+
+    def get(parameters)
+      @cache.getset(parameters) { yield }
+    end
+  end
+
+  class NoCache
+    def initialize(size, ttl) end
+
+    def get(statement)
+      yield
+    end
+  end
+
+  include LogStash::PluginMixins::JdbcStreaming
+
+  config_name "jdbc_streaming"
+
+  # Statement to execute.
+  # To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id"
+  config :statement, :validate => :string, :required => true
+
+  # Hash of query parameter, for example `{ "id" => "id_field" }`
+  config :parameters, :validate => :hash, :default => {}
+
+  # Define the target field to store the extracted result(s)
+  # Field is overwritten if exists
+  config :target, :validate => :string, :required => true
+
+  # Define a default object to use when lookup fails to return a matching row.
+  # ensure that the key names of this object match the columns from the statement
+  config :default_hash, :validate => :hash, :default => {}
+
+  # Append values to the `tags` field if sql error occured
+  config :tag_on_failure, :validate => :array, :default => ["_jdbcstreamingfailure"]
+
+  # Append values to the `tags` field if no record was found and default values were used
+  config :tag_on_default_use, :validate => :array, :default => ["_jdbcstreamingdefaultsused"]
+
+  # Enable or disable caching, boolean true or false, defaults to true
+  config :use_cache, :validate => :boolean, :default => true
+
+  # The minimum number of seconds any entry should remain in the cache, defaults to 5 seconds
+  # A numeric value, you can use decimals for example `{ "cache_expiration" => 0.25 }`
+  # If there are transient jdbc errors the cache will store empty results for a given
+  # parameter set and bypass the jbdc lookup, this merges the default_hash into the event, until
+  # the cache entry expires, then the jdbc lookup will be tried again for the same parameters
+  # Conversely, while the cache contains valid results any external problem that would cause
+  # jdbc errors, will not be noticed for the cache_expiration period.
+  config :cache_expiration, :validate => :number, :default => 5.0
+
+  # The maximum number of cache entries are stored, defaults to 500 entries
+  # The least recently used entry will be evicted
+  config :cache_size, :validate => :number, :default => 500
+
+  # ----------------------------------------
+  public
+
+  def register
+    convert_config_options
+    prepare_connected_jdbc_cache
+  end
+
+  def filter(event)
+    result = cache_lookup(event) # should return a JdbcCachePayload
+
+    if result.failed?
+      tag_failure(event)
+    end
+
+    if result.empty?
+      tag_default(event)
+      process_event(event, @default_array)
+    else
+      process_event(event, result.payload)
+    end
+  end
+
+  # ----------------------------------------
+  private
+
+  def cache_lookup(event)
+    params = prepare_parameters_from_event(event)
+    @cache.get(params) do
+      result = CachePayload.new
+      begin
+        query = @database[@statement, params] # returns a dataset
+        @logger.debug? && @logger.debug("Executing JDBC query", :statement => @statement, :parameters => params)
+        query.all do |row|
+          result.push row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
+        end
+      rescue ::Sequel::Error => e
+        # all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
+        result.failed!
+        @logger.warn? && @logger.warn("Exception when executing JDBC query", :exception => e)
+      end
+      # if either of: no records or a Sequel exception occurs the payload is
+      # empty and the default can be substituted later.
+      result
+    end
+  end
+
+  def prepare_parameters_from_event(event)
+    @symbol_parameters.inject({}) do |hash,(k,v)|
+      value = event.get(event.sprintf(v))
+      hash[k] = value.is_a?(::LogStash::Timestamp) ? value.time : value
+      hash
+    end
+  end
+
+  def tag_failure(event)
+    @tag_on_failure.each do |tag|
+      event.tag(tag)
+    end
+  end
+
+  def tag_default(event)
+    @tag_on_default_use.each do |tag|
+      event.tag(tag)
+    end
+  end
+
+  def process_event(event, value)
+    # use deep clone here so other filter function don't taint the cached payload by reference
+    event.set(@target, ::LogStash::Util.deep_clone(value))
+    filter_matched(event)
+  end
+
+  def convert_config_options
+    # create these object once they will be cloned for every filter call anyway,
+    # lets not create a new object for each
+    @symbol_parameters = @parameters.inject({}) {|hash,(k,v)| hash[k.to_sym] = v ; hash }
+    @default_array = [@default_hash]
+  end
+
+  def prepare_connected_jdbc_cache
+    klass = @use_cache ? RowCache : NoCache
+    @cache = klass.new(@cache_size, @cache_expiration)
+    prepare_jdbc_connection
+  end
+end end end # class LogStash::Filters::Jdbc

data/lib/logstash/plugin_mixins/jdbc_streaming.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+require "logstash/config/mixin"
+
+# Tentative of abstracting JDBC logic to a mixin
+# for potential reuse in other plugins (input/output)
+module LogStash module PluginMixins module JdbcStreaming
+
+  # This method is called when someone includes this module
+  def self.included(base)
+    # Add these methods to the 'base' given.
+    base.extend(self)
+    base.setup_jdbc_config
+  end
+
+  public
+  def setup_jdbc_config
+    # JDBC driver library path to third party driver library.
+    config :jdbc_driver_library, :validate => :path
+
+    # JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver"
+    config :jdbc_driver_class, :validate => :string, :required => true
+
+    # JDBC connection string
+    config :jdbc_connection_string, :validate => :string, :required => true
+
+    # JDBC user
+    config :jdbc_user, :validate => :string
+
+    # JDBC password
+    config :jdbc_password, :validate => :password
+
+    # Connection pool configuration.
+    # Validate connection before use.
+    config :jdbc_validate_connection, :validate => :boolean, :default => false
+
+    # Connection pool configuration.
+    # How often to validate a connection (in seconds)
+    config :jdbc_validation_timeout, :validate => :number, :default => 3600
+  end
+
+  public
+  def prepare_jdbc_connection
+    require "sequel"
+    require "sequel/adapters/jdbc"
+    require "java"
+    require @jdbc_driver_library if @jdbc_driver_library
+    Sequel::JDBC.load_driver(@jdbc_driver_class)
+    @database = Sequel.connect(@jdbc_connection_string, :user=> @jdbc_user, :password=>  @jdbc_password.nil? ? nil : @jdbc_password.value)
+    if @jdbc_validate_connection
+      @database.extension(:connection_validator)
+      @database.pool.connection_validation_timeout = @jdbc_validation_timeout
+    end
+    begin
+      @database.test_connection
+    rescue Sequel::DatabaseConnectionError => e
+      #TODO return false and let the plugin raise a LogStash::ConfigurationError
+      raise e
+    end
+  end # def prepare_jdbc_connection
+end end end

data/logstash-filter-jdbc_streaming.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-filter-jdbc_streaming'
+  s.version         = '1.0.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "This filter executes a SQL query and store the result set in the event."
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elastic"]
+  s.email           = 'info@elastic.co'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths   = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'sequel'
+  s.add_runtime_dependency 'lru_redux' # lru cache with ttl
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'jdbc-derby'
+  s.add_development_dependency 'jdbc-mysql'
+end

data/spec/filters/jdbc_streaming_spec.rb

@@ -0,0 +1,277 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc_streaming"
+require 'jdbc/derby'
+require "sequel"
+require "sequel/adapters/jdbc"
+
+module LogStash module Filters
+  class TestJdbcStreaming < JdbcStreaming
+    attr_reader :database
+  end
+
+  describe JdbcStreaming do
+    #Use embedded Derby for tests
+    ::Jdbc::Derby.load_driver
+
+    ENV["TZ"] = "Etc/UTC"
+    describe "plugin level execution" do
+      let(:mixin_settings) do
+        { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+          "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"}
+      end
+      let(:plugin) { JdbcStreaming.new(mixin_settings.merge(settings)) }
+      let (:db) do
+        ::Sequel.connect(mixin_settings['jdbc_connection_string'], :user=> nil, :password=> nil)
+      end
+      let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
+      let(:cache_expiration) { 3.0 }
+      let(:use_cache) { true }
+      let(:cache_size) { 10 }
+      let(:statement) { "SELECT name, location FROM reference_table WHERE ip = :ip" }
+      let(:settings) do
+        {
+          "statement" => statement,
+          "parameters" => {"ip" => "ip"},
+          "target" => "server",
+          "use_cache" => use_cache,
+          "cache_expiration" => cache_expiration,
+          "cache_size" => cache_size,
+          "tag_on_failure" => ["lookup_failed"],
+          "tag_on_default_use" => ["default_used_instead"],
+          "default_hash" => {"name" => "unknown", "location" => "unknown"}
+        }
+      end
+
+      before :each do
+        db.create_table :reference_table do
+          String :ip
+          String  :name
+          String  :location
+        end
+        db[:reference_table].insert(:ip => "10.1.1.1", :name => "ldn-server-1", :location => "LDN-2-3-4")
+        db[:reference_table].insert(:ip => "10.2.1.1", :name => "nyc-server-1", :location => "NYC-5-2-8")
+        db[:reference_table].insert(:ip => "10.3.1.1", :name => "mv-server-1", :location => "MV-9-6-4")
+        plugin.register
+      end
+
+      after :each do
+        db.drop_table(:reference_table)
+      end
+
+      describe "found record - uses row" do
+        let(:ipaddr)    { "10.1.1.1" }
+
+        it "fills in the target" do
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"name" => "ldn-server-1", "location" => "LDN-2-3-4"}])
+          expect(event.get("tags") || []).not_to include("lookup_failed")
+          expect(event.get("tags") || []).not_to include("default_used_instead")
+        end
+      end
+
+      describe "missing record - uses default" do
+        let(:ipaddr)    { "192.168.1.1" }
+
+        it "fills in the target with the default" do
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+          expect(event.get("tags") & ["lookup_failed", "default_used_instead"]).to eq(["default_used_instead"])
+        end
+      end
+
+      describe "database error - uses default" do
+        let(:ipaddr)    { "10.1.1.1" }
+        let(:statement) { "SELECT name, location FROM reference_table WHERE ip = :address" }
+        it "fills in the target with the default" do
+          plugin.filter(event)
+          expect(event.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+          expect(event.get("tags") & ["lookup_failed", "default_used_instead"]).to eq(["lookup_failed", "default_used_instead"])
+        end
+      end
+
+      context "when fetching from cache" do
+        let(:plugin) { TestJdbcStreaming.new(mixin_settings.merge(settings)) }
+        let(:events) do
+          5.times.map{|i| ::LogStash::Event.new("message" => "some other text #{i}", "ip" => ipaddr) }
+        end
+        let(:call_count) { 1 }
+        before(:each) do
+          expect(plugin.database).to receive(:[]).exactly(call_count).times.and_call_original
+          plugin.filter(event)
+        end
+
+        describe "found record - caches row" do
+          let(:ipaddr)    { "10.1.1.1" }
+          it "calls the database once then uses the cache" do
+            expect(event.get("server")).to eq([{"name" => "ldn-server-1", "location" => "LDN-2-3-4"}])
+            expect(event.get("tags") || []).not_to include("lookup_failed")
+            expect(event.get("tags") || []).not_to include("default_used_instead")
+            events.each do |evt|
+              plugin.filter(evt)
+              expect(evt.get("server")).to eq([{"name" => "ldn-server-1", "location" => "LDN-2-3-4"}])
+            end
+          end
+        end
+
+        describe "missing record - uses default" do
+          let(:ipaddr)    { "10.10.1.1" }
+          it "calls the database once then uses the cache" do
+            expect(event.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+            expect(event.get("tags") & ["lookup_failed", "default_used_instead"]).to eq(["default_used_instead"])
+            events.each do |evt|
+              plugin.filter(evt)
+              expect(evt.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+            end
+          end
+        end
+
+        context "extremely small cache expiration" do
+          describe "found record - cache always expires" do
+            let(:ipaddr)    { "10.1.1.1" }
+            let(:call_count) { 6 }
+            let(:cache_expiration) { 0.0000001 }
+            it "calls the database each time because cache entry expired" do
+              expect(event.get("server")).to eq([{"name" => "ldn-server-1", "location" => "LDN-2-3-4"}])
+              expect(event.get("tags") || []).not_to include("lookup_failed")
+              expect(event.get("tags") || []).not_to include("default_used_instead")
+              events.each do |evt|
+                plugin.filter(evt)
+                expect(evt.get("server")).to eq([{"name" => "ldn-server-1", "location" => "LDN-2-3-4"}])
+              end
+            end
+          end
+        end
+
+        context "when cache is disabled" do
+          let(:call_count) { 6 }
+          let(:use_cache) { false }
+          describe "database is always called" do
+            let(:ipaddr)    { "10.1.1.1" }
+            it "calls the database each time" do
+              expect(event.get("server")).to eq([{"name" => "ldn-server-1", "location" => "LDN-2-3-4"}])
+              expect(event.get("tags") || []).not_to include("lookup_failed")
+              expect(event.get("tags") || []).not_to include("default_used_instead")
+              events.each do |evt|
+                plugin.filter(evt)
+                expect(evt.get("server")).to eq([{"name" => "ldn-server-1", "location" => "LDN-2-3-4"}])
+              end
+            end
+          end
+
+          describe "database is always called but record is missing and default is used" do
+            let(:ipaddr)    { "10.11.1.1" }
+            it "calls the database each time" do
+              expect(event.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+              expect(event.get("tags") & ["lookup_failed", "default_used_instead"]).to eq(["default_used_instead"])
+              events.each do |evt|
+                plugin.filter(evt)
+                expect(evt.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+              end
+            end
+          end
+        end
+      end
+    end
+
+    describe "All default - Retrieve a value from database" do
+      let(:config) do <<-CONFIG
+        filter {
+          jdbc_streaming {
+            jdbc_driver_class => "org.apache.derby.jdbc.EmbeddedDriver"
+            jdbc_connection_string => "jdbc:derby:memory:testdb;create=true"
+            statement => "SELECT 'from_database' FROM SYSIBM.SYSDUMMY1"
+            target => "new_field"
+          }
+        }
+      CONFIG
+      end
+
+      sample("message" => "some text") do
+        expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
+      end
+    end
+
+    describe "Named column - Retrieve a value from database" do
+      let(:config) do <<-CONFIG
+        filter {
+          jdbc_streaming {
+            jdbc_driver_class => "org.apache.derby.jdbc.EmbeddedDriver"
+            jdbc_connection_string => "jdbc:derby:memory:testdb;create=true"
+            statement => "SELECT 'from_database' as col_1 FROM SYSIBM.SYSDUMMY1"
+            target => "new_field"
+          }
+        }
+      CONFIG
+      end
+
+      sample("message" => "some text") do
+        expect(subject.get('new_field')).to eq([{"col_1" => 'from_database'}])
+      end
+    end
+
+    describe "Using string parameters - Retrieve a value from database" do
+      let(:config) do <<-CONFIG
+        filter {
+          jdbc_streaming {
+            jdbc_driver_class => "org.apache.derby.jdbc.EmbeddedDriver"
+            jdbc_connection_string => "jdbc:derby:memory:testdb;create=true"
+            statement => "SELECT 'from_database' FROM SYSIBM.SYSDUMMY1 WHERE '1' = :param"
+            parameters => { "param" => "param_field"}
+            target => "new_field"
+          }
+        }
+      CONFIG
+      end
+
+      sample("message" => "some text", "param_field" => "1") do
+        expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
+      end
+
+      sample("message" => "some text", "param_field" => "2") do
+        expect(subject.get('new_field').nil?)
+      end
+    end
+
+    describe "Using integer parameters" do
+      let(:config) do <<-CONFIG
+        filter {
+          jdbc_streaming {
+            jdbc_driver_class => "org.apache.derby.jdbc.EmbeddedDriver"
+            jdbc_connection_string => "jdbc:derby:memory:testdb;create=true"
+            statement => "SELECT 'from_database' FROM SYSIBM.SYSDUMMY1 WHERE 1 = :param"
+            parameters => { "param" => "param_field"}
+            target => "new_field"
+          }
+        }
+      CONFIG
+      end
+
+      sample("message" => "some text", "param_field" => 1) do
+        expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
+      end
+
+      sample("message" => "some text", "param_field" => "1") do
+        expect(subject.get('new_field').nil?)
+      end
+    end
+
+    describe "Using timestamp parameter" do
+      let(:config) do <<-CONFIG
+        filter {
+          jdbc_streaming {
+            jdbc_driver_class => "org.apache.derby.jdbc.EmbeddedDriver"
+            jdbc_connection_string => "jdbc:derby:memory:testdb;create=true"
+            statement => "SELECT 'from_database' FROM SYSIBM.SYSDUMMY1 WHERE {fn TIMESTAMPDIFF( SQL_TSI_DAY, {t :param}, current_timestamp)} = 0"
+            parameters => { "param" => "@timestamp"}
+            target => "new_field"
+          }
+        }
+      CONFIG
+      end
+
+      sample("message" => "some text") do
+        expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
+      end
+    end
+  end
+end end

data/spec/integration/jdbcstreaming_spec.rb

@@ -0,0 +1,154 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/jdbc_streaming"
+require 'jdbc/mysql'
+require "sequel"
+require "sequel/adapters/jdbc"
+
+module LogStash module Filters
+  class TestJdbcStreaming < JdbcStreaming
+    attr_reader :database
+  end
+
+  describe JdbcStreaming, :integration => true do
+    # Use MySql for integration tests
+    ::Jdbc::MySQL.load_driver
+
+    ENV["TZ"] = "Etc/UTC"
+    let(:mixin_settings) do
+      { "jdbc_user" => "root", "jdbc_driver_class" => "com.mysql.jdbc.Driver",
+        "jdbc_connection_string" => "jdbc:mysql://localhost/jdbc_streaming_db?user=root"}
+    end
+    let(:settings) { {} }
+    let(:plugin) { JdbcStreaming.new(mixin_settings.merge(settings)) }
+    let (:db) do
+      ::Sequel.connect(mixin_settings['jdbc_connection_string'])
+    end
+    let(:event)      { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
+    let(:cache_expiration) { 3.0 }
+    let(:use_cache) { true }
+    let(:cache_size) { 10 }
+    let(:statement) { "SELECT name, location FROM reference_table WHERE ip = :ip" }
+    let(:settings) do
+      {
+        "statement" => statement,
+        "parameters" => {"ip" => "ip"},
+        "target" => "server",
+        "use_cache" => use_cache,
+        "cache_expiration" => cache_expiration,
+        "cache_size" => cache_size,
+        "tag_on_failure" => ["lookup_failed"],
+        "tag_on_default_use" => ["default_used_instead"],
+        "default_hash" => {"name" => "unknown", "location" => "unknown"}
+      }
+    end
+    let(:ipaddr)    { "10.#{idx}.1.1" }
+
+    before :each do
+      db.create_table :reference_table do
+        String :ip
+        String  :name
+        String  :location
+      end
+      1.upto(250) do |i|
+        db[:reference_table].insert(:ip => "10.#{i}.1.1", :name => "ldn-server-#{i}", :location => "LDN-#{i}-2-3")
+      end
+      plugin.register
+    end
+
+    after(:each) { db.drop_table(:reference_table) }
+
+    describe "found record - uses row" do
+      let(:idx) { 200 }
+
+      it "fills in the target" do
+        plugin.filter(event)
+        expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+        expect((event.get("tags") || []) & ["lookup_failed", "default_used_instead"]).to be_empty
+      end
+    end
+
+    context "when fetching from cache" do
+      let(:plugin) { TestJdbcStreaming.new(mixin_settings.merge(settings)) }
+      let(:events) do
+        5.times.map{|i| ::LogStash::Event.new("message" => "some other text #{i}", "ip" => ipaddr) }
+      end
+      let(:call_count) { 1 }
+      before(:each) do
+        expect(plugin.database).to receive(:[]).exactly(call_count).times.and_call_original
+        plugin.filter(event)
+      end
+
+      describe "found record - caches row" do
+        let(:idx)    { "42" }
+        it "calls the database once then uses the cache" do
+          expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+          expect(event.get("tags") || []).not_to include("lookup_failed")
+          expect(event.get("tags") || []).not_to include("default_used_instead")
+          events.each do |evt|
+            plugin.filter(evt)
+            expect(evt.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+          end
+        end
+      end
+
+      describe "missing record - uses default" do
+        let(:idx)    { "252" }
+        it "calls the database once then uses the cache" do
+          expect(event.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+          expect(event.get("tags") & ["lookup_failed", "default_used_instead"]).to eq(["default_used_instead"])
+          events.each do |evt|
+            plugin.filter(evt)
+            expect(evt.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+          end
+        end
+      end
+
+      context "extremely small cache expiration" do
+        describe "found record - cache always expires" do
+          let(:idx)    { "10" }
+          let(:call_count) { 6 }
+          let(:cache_expiration) { 0.0000001 }
+          it "calls the database each time because cache entry expired" do
+            expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+            expect(event.get("tags") || []).not_to include("lookup_failed")
+            expect(event.get("tags") || []).not_to include("default_used_instead")
+            events.each do |evt|
+              plugin.filter(evt)
+              expect(evt.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+            end
+          end
+        end
+      end
+
+      context "when cache is disabled" do
+        let(:call_count) { 6 }
+        let(:use_cache) { false }
+        describe "database is always called" do
+          let(:idx)    { "1" }
+          it "calls the database each time" do
+            expect(event.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+            expect(event.get("tags") || []).not_to include("lookup_failed")
+            expect(event.get("tags") || []).not_to include("default_used_instead")
+            events.each do |evt|
+              plugin.filter(evt)
+              expect(evt.get("server")).to eq([{"name" => "ldn-server-#{idx}", "location" => "LDN-#{idx}-2-3"}])
+            end
+          end
+        end
+
+        describe "database is always called but record is missing and default is used" do
+          let(:idx)    { "251" }
+          it "calls the database each time" do
+            expect(event.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+            expect(event.get("tags") & ["lookup_failed", "default_used_instead"]).to eq(["default_used_instead"])
+            events.each do |evt|
+              plugin.filter(evt)
+              expect(evt.get("server")).to eq([{"name" => "unknown", "location" => "unknown"}])
+            end
+          end
+        end
+      end
+    end
+  end
+
+end end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-jdbc_streaming
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-03-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: sequel
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: lru_redux
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: jdbc-derby
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: jdbc-mysql
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/jdbc_streaming.rb
+- lib/logstash/plugin_mixins/jdbc_streaming.rb
+- logstash-filter-jdbc_streaming.gemspec
+- spec/filters/jdbc_streaming_spec.rb
+- spec/integration/jdbcstreaming_spec.rb
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: This filter executes a SQL query and store the result set in the event.
+test_files:
+- spec/filters/jdbc_streaming_spec.rb
+- spec/integration/jdbcstreaming_spec.rb