logstash-filter-jdbc_streaming 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a1ec4d5c58f6ce550226691dca0007a13c4b136
-  data.tar.gz: a3a51d17ea4a55cf57e74537a783c651cd737286
+  metadata.gz: bd1cf10a30bedfabbff7d098941308be4e357b0b
+  data.tar.gz: c6ee35fd7b5dd5f8d606f5920b4b475a88a4983f
 SHA512:
-  metadata.gz: 6ce0c39c40c7f0e93776bd9abdf185cc3780d96d3742e051488db497175fc3929a9bfe04943bde7e26b6f0d1a915363e3e00e28a3c77d6de6242ec2ddadc30e7
-  data.tar.gz: adf5c47acdac61a900d4f2288aa466bbcbdc28165ab7d6d28fc666eedb1e294024348127f789f69ec7f4c2b62d11c86491c627149d69d11f083672953f25b2b0
+  metadata.gz: 34524e8efeb0442e60f7f26c54c1ae2e890f6e0bb40665dcb5dff86290bfe96902de118f14eab0690eb3ceca8d5bfe8d5620ed70628195b1086e2cd7aaa3692c
+  data.tar.gz: 9ccea4c9ed6fdec97a4898666548b9e9951c1f814352c56af5f0c61151a248a8c2c7f9a8036505d4bb086796da1e90ea5896fd8f990e97d5321454496c0db81f

data/Gemfile

@@ -1,2 +1,11 @@
 source 'https://rubygems.org'
+
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,226 @@
+:plugin: jdbc_streaming
+:type: filter
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Jdbc_streaming filter plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This filter executes a SQL query and store the result set in the field
+specified as `target`.
+It will cache the results locally in an LRU cache with expiry
+
+For example you can load a row based on an id from in the event
+
+[source,ruby]
+filter {
+  jdbc_streaming {
+    jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar"
+    jdbc_driver_class => "com.mysql.jdbc.Driver"
+    jdbc_connection_string => ""jdbc:mysql://localhost:3306/mydatabase"
+    jdbc_user => "me"
+    jdbc_password => "secret"
+    statement => "select * from WORLD.COUNTRY WHERE Code = :code"
+    parameters => { "code" => "country_code"}
+    target => "country_details"
+  }
+}
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Jdbc_streaming Filter Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-cache_expiration>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-cache_size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-default_hash>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-jdbc_connection_string>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-jdbc_driver_class>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-jdbc_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-jdbc_user>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-jdbc_validate_connection>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-jdbc_validation_timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-statement>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-tag_on_default_use>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-use_cache>> |<<boolean,boolean>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+filter plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-cache_expiration"]
+===== `cache_expiration` 
+
+  * Value type is <<number,number>>
+  * Default value is `5.0`
+
+The minimum number of seconds any entry should remain in the cache, defaults to 5 seconds
+A numeric value, you can use decimals for example `{ "cache_expiration" => 0.25 }`
+If there are transient jdbc errors the cache will store empty results for a given
+parameter set and bypass the jbdc lookup, this merges the default_hash into the event, until
+the cache entry expires, then the jdbc lookup will be tried again for the same parameters
+Conversely, while the cache contains valid results any external problem that would cause
+jdbc errors, will not be noticed for the cache_expiration period.
+
+[id="plugins-{type}s-{plugin}-cache_size"]
+===== `cache_size` 
+
+  * Value type is <<number,number>>
+  * Default value is `500`
+
+The maximum number of cache entries are stored, defaults to 500 entries
+The least recently used entry will be evicted
+
+[id="plugins-{type}s-{plugin}-default_hash"]
+===== `default_hash` 
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Define a default object to use when lookup fails to return a matching row.
+ensure that the key names of this object match the columns from the statement
+
+[id="plugins-{type}s-{plugin}-jdbc_connection_string"]
+===== `jdbc_connection_string` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+JDBC connection string
+
+[id="plugins-{type}s-{plugin}-jdbc_driver_class"]
+===== `jdbc_driver_class` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver"
+
+[id="plugins-{type}s-{plugin}-jdbc_driver_library"]
+===== `jdbc_driver_library` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+Tentative of abstracting JDBC logic to a mixin
+for potential reuse in other plugins (input/output)
+This method is called when someone includes this module
+Add these methods to the 'base' given.
+JDBC driver library path to third party driver library.
+
+[id="plugins-{type}s-{plugin}-jdbc_password"]
+===== `jdbc_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+JDBC password
+
+[id="plugins-{type}s-{plugin}-jdbc_user"]
+===== `jdbc_user` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+JDBC user
+
+[id="plugins-{type}s-{plugin}-jdbc_validate_connection"]
+===== `jdbc_validate_connection` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Connection pool configuration.
+Validate connection before use.
+
+[id="plugins-{type}s-{plugin}-jdbc_validation_timeout"]
+===== `jdbc_validation_timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `3600`
+
+Connection pool configuration.
+How often to validate a connection (in seconds)
+
+[id="plugins-{type}s-{plugin}-parameters"]
+===== `parameters` 
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+Hash of query parameter, for example `{ "id" => "id_field" }`
+
+[id="plugins-{type}s-{plugin}-statement"]
+===== `statement` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Statement to execute.
+To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id"
+
+[id="plugins-{type}s-{plugin}-tag_on_default_use"]
+===== `tag_on_default_use` 
+
+  * Value type is <<array,array>>
+  * Default value is `["_jdbcstreamingdefaultsused"]`
+
+Append values to the `tags` field if no record was found and default values were used
+
+[id="plugins-{type}s-{plugin}-tag_on_failure"]
+===== `tag_on_failure` 
+
+  * Value type is <<array,array>>
+  * Default value is `["_jdbcstreamingfailure"]`
+
+Append values to the `tags` field if sql error occured
+
+[id="plugins-{type}s-{plugin}-target"]
+===== `target` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Define the target field to store the extracted result(s)
+Field is overwritten if exists
+
+[id="plugins-{type}s-{plugin}-use_cache"]
+===== `use_cache` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Enable or disable caching, boolean true or false, defaults to true
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/logstash-filter-jdbc_streaming.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-filter-jdbc_streaming'
-  s.version         = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter executes a SQL query and store the result set in the event."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths   = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
    # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-jdbc_streaming
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-27 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -110,6 +110,7 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- docs/index.asciidoc
 - lib/logstash/filters/jdbc_streaming.rb
 - lib/logstash/plugin_mixins/jdbc_streaming.rb
 - logstash-filter-jdbc_streaming.gemspec