logstash-filter-jdbc_streaming 1.0.0 → 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0a1ec4d5c58f6ce550226691dca0007a13c4b136
4
- data.tar.gz: a3a51d17ea4a55cf57e74537a783c651cd737286
3
+ metadata.gz: bd1cf10a30bedfabbff7d098941308be4e357b0b
4
+ data.tar.gz: c6ee35fd7b5dd5f8d606f5920b4b475a88a4983f
5
5
  SHA512:
6
- metadata.gz: 6ce0c39c40c7f0e93776bd9abdf185cc3780d96d3742e051488db497175fc3929a9bfe04943bde7e26b6f0d1a915363e3e00e28a3c77d6de6242ec2ddadc30e7
7
- data.tar.gz: adf5c47acdac61a900d4f2288aa466bbcbdc28165ab7d6d28fc666eedb1e294024348127f789f69ec7f4c2b62d11c86491c627149d69d11f083672953f25b2b0
6
+ metadata.gz: 34524e8efeb0442e60f7f26c54c1ae2e890f6e0bb40665dcb5dff86290bfe96902de118f14eab0690eb3ceca8d5bfe8d5620ed70628195b1086e2cd7aaa3692c
7
+ data.tar.gz: 9ccea4c9ed6fdec97a4898666548b9e9951c1f814352c56af5f0c61151a248a8c2c7f9a8036505d4bb086796da1e90ea5896fd8f990e97d5321454496c0db81f
data/Gemfile CHANGED
@@ -1,2 +1,11 @@
1
1
  source 'https://rubygems.org'
2
+
2
3
  gemspec
4
+
5
+ logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
6
+ use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
7
+
8
+ if Dir.exist?(logstash_path) && use_logstash_source
9
+ gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
10
+ gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
11
+ end
@@ -0,0 +1,226 @@
1
+ :plugin: jdbc_streaming
2
+ :type: filter
3
+
4
+ ///////////////////////////////////////////
5
+ START - GENERATED VARIABLES, DO NOT EDIT!
6
+ ///////////////////////////////////////////
7
+ :version: %VERSION%
8
+ :release_date: %RELEASE_DATE%
9
+ :changelog_url: %CHANGELOG_URL%
10
+ :include_path: ../../../../logstash/docs/include
11
+ ///////////////////////////////////////////
12
+ END - GENERATED VARIABLES, DO NOT EDIT!
13
+ ///////////////////////////////////////////
14
+
15
+ [id="plugins-{type}-{plugin}"]
16
+
17
+ === Jdbc_streaming filter plugin
18
+
19
+ include::{include_path}/plugin_header.asciidoc[]
20
+
21
+ ==== Description
22
+
23
+ This filter executes a SQL query and store the result set in the field
24
+ specified as `target`.
25
+ It will cache the results locally in an LRU cache with expiry
26
+
27
+ For example you can load a row based on an id from in the event
28
+
29
+ [source,ruby]
30
+ filter {
31
+ jdbc_streaming {
32
+ jdbc_driver_library => "/path/to/mysql-connector-java-5.1.34-bin.jar"
33
+ jdbc_driver_class => "com.mysql.jdbc.Driver"
34
+ jdbc_connection_string => ""jdbc:mysql://localhost:3306/mydatabase"
35
+ jdbc_user => "me"
36
+ jdbc_password => "secret"
37
+ statement => "select * from WORLD.COUNTRY WHERE Code = :code"
38
+ parameters => { "code" => "country_code"}
39
+ target => "country_details"
40
+ }
41
+ }
42
+
43
+
44
+ [id="plugins-{type}s-{plugin}-options"]
45
+ ==== Jdbc_streaming Filter Configuration Options
46
+
47
+ This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
48
+
49
+ [cols="<,<,<",options="header",]
50
+ |=======================================================================
51
+ |Setting |Input type|Required
52
+ | <<plugins-{type}s-{plugin}-cache_expiration>> |<<number,number>>|No
53
+ | <<plugins-{type}s-{plugin}-cache_size>> |<<number,number>>|No
54
+ | <<plugins-{type}s-{plugin}-default_hash>> |<<hash,hash>>|No
55
+ | <<plugins-{type}s-{plugin}-jdbc_connection_string>> |<<string,string>>|Yes
56
+ | <<plugins-{type}s-{plugin}-jdbc_driver_class>> |<<string,string>>|Yes
57
+ | <<plugins-{type}s-{plugin}-jdbc_driver_library>> |a valid filesystem path|No
58
+ | <<plugins-{type}s-{plugin}-jdbc_password>> |<<password,password>>|No
59
+ | <<plugins-{type}s-{plugin}-jdbc_user>> |<<string,string>>|No
60
+ | <<plugins-{type}s-{plugin}-jdbc_validate_connection>> |<<boolean,boolean>>|No
61
+ | <<plugins-{type}s-{plugin}-jdbc_validation_timeout>> |<<number,number>>|No
62
+ | <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
63
+ | <<plugins-{type}s-{plugin}-statement>> |<<string,string>>|Yes
64
+ | <<plugins-{type}s-{plugin}-tag_on_default_use>> |<<array,array>>|No
65
+ | <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
66
+ | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|Yes
67
+ | <<plugins-{type}s-{plugin}-use_cache>> |<<boolean,boolean>>|No
68
+ |=======================================================================
69
+
70
+ Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
71
+ filter plugins.
72
+
73
+ &nbsp;
74
+
75
+ [id="plugins-{type}s-{plugin}-cache_expiration"]
76
+ ===== `cache_expiration`
77
+
78
+ * Value type is <<number,number>>
79
+ * Default value is `5.0`
80
+
81
+ The minimum number of seconds any entry should remain in the cache, defaults to 5 seconds
82
+ A numeric value, you can use decimals for example `{ "cache_expiration" => 0.25 }`
83
+ If there are transient jdbc errors the cache will store empty results for a given
84
+ parameter set and bypass the jbdc lookup, this merges the default_hash into the event, until
85
+ the cache entry expires, then the jdbc lookup will be tried again for the same parameters
86
+ Conversely, while the cache contains valid results any external problem that would cause
87
+ jdbc errors, will not be noticed for the cache_expiration period.
88
+
89
+ [id="plugins-{type}s-{plugin}-cache_size"]
90
+ ===== `cache_size`
91
+
92
+ * Value type is <<number,number>>
93
+ * Default value is `500`
94
+
95
+ The maximum number of cache entries are stored, defaults to 500 entries
96
+ The least recently used entry will be evicted
97
+
98
+ [id="plugins-{type}s-{plugin}-default_hash"]
99
+ ===== `default_hash`
100
+
101
+ * Value type is <<hash,hash>>
102
+ * Default value is `{}`
103
+
104
+ Define a default object to use when lookup fails to return a matching row.
105
+ ensure that the key names of this object match the columns from the statement
106
+
107
+ [id="plugins-{type}s-{plugin}-jdbc_connection_string"]
108
+ ===== `jdbc_connection_string`
109
+
110
+ * This is a required setting.
111
+ * Value type is <<string,string>>
112
+ * There is no default value for this setting.
113
+
114
+ JDBC connection string
115
+
116
+ [id="plugins-{type}s-{plugin}-jdbc_driver_class"]
117
+ ===== `jdbc_driver_class`
118
+
119
+ * This is a required setting.
120
+ * Value type is <<string,string>>
121
+ * There is no default value for this setting.
122
+
123
+ JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver"
124
+
125
+ [id="plugins-{type}s-{plugin}-jdbc_driver_library"]
126
+ ===== `jdbc_driver_library`
127
+
128
+ * Value type is <<path,path>>
129
+ * There is no default value for this setting.
130
+
131
+ Tentative of abstracting JDBC logic to a mixin
132
+ for potential reuse in other plugins (input/output)
133
+ This method is called when someone includes this module
134
+ Add these methods to the 'base' given.
135
+ JDBC driver library path to third party driver library.
136
+
137
+ [id="plugins-{type}s-{plugin}-jdbc_password"]
138
+ ===== `jdbc_password`
139
+
140
+ * Value type is <<password,password>>
141
+ * There is no default value for this setting.
142
+
143
+ JDBC password
144
+
145
+ [id="plugins-{type}s-{plugin}-jdbc_user"]
146
+ ===== `jdbc_user`
147
+
148
+ * Value type is <<string,string>>
149
+ * There is no default value for this setting.
150
+
151
+ JDBC user
152
+
153
+ [id="plugins-{type}s-{plugin}-jdbc_validate_connection"]
154
+ ===== `jdbc_validate_connection`
155
+
156
+ * Value type is <<boolean,boolean>>
157
+ * Default value is `false`
158
+
159
+ Connection pool configuration.
160
+ Validate connection before use.
161
+
162
+ [id="plugins-{type}s-{plugin}-jdbc_validation_timeout"]
163
+ ===== `jdbc_validation_timeout`
164
+
165
+ * Value type is <<number,number>>
166
+ * Default value is `3600`
167
+
168
+ Connection pool configuration.
169
+ How often to validate a connection (in seconds)
170
+
171
+ [id="plugins-{type}s-{plugin}-parameters"]
172
+ ===== `parameters`
173
+
174
+ * Value type is <<hash,hash>>
175
+ * Default value is `{}`
176
+
177
+ Hash of query parameter, for example `{ "id" => "id_field" }`
178
+
179
+ [id="plugins-{type}s-{plugin}-statement"]
180
+ ===== `statement`
181
+
182
+ * This is a required setting.
183
+ * Value type is <<string,string>>
184
+ * There is no default value for this setting.
185
+
186
+ Statement to execute.
187
+ To use parameters, use named parameter syntax, for example "SELECT * FROM MYTABLE WHERE ID = :id"
188
+
189
+ [id="plugins-{type}s-{plugin}-tag_on_default_use"]
190
+ ===== `tag_on_default_use`
191
+
192
+ * Value type is <<array,array>>
193
+ * Default value is `["_jdbcstreamingdefaultsused"]`
194
+
195
+ Append values to the `tags` field if no record was found and default values were used
196
+
197
+ [id="plugins-{type}s-{plugin}-tag_on_failure"]
198
+ ===== `tag_on_failure`
199
+
200
+ * Value type is <<array,array>>
201
+ * Default value is `["_jdbcstreamingfailure"]`
202
+
203
+ Append values to the `tags` field if sql error occured
204
+
205
+ [id="plugins-{type}s-{plugin}-target"]
206
+ ===== `target`
207
+
208
+ * This is a required setting.
209
+ * Value type is <<string,string>>
210
+ * There is no default value for this setting.
211
+
212
+ Define the target field to store the extracted result(s)
213
+ Field is overwritten if exists
214
+
215
+ [id="plugins-{type}s-{plugin}-use_cache"]
216
+ ===== `use_cache`
217
+
218
+ * Value type is <<boolean,boolean>>
219
+ * Default value is `true`
220
+
221
+ Enable or disable caching, boolean true or false, defaults to true
222
+
223
+
224
+
225
+ [id="plugins-{type}s-{plugin}-common-options"]
226
+ include::{include_path}/{type}.asciidoc[]
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-filter-jdbc_streaming'
3
- s.version = '1.0.0'
3
+ s.version = '1.0.1'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "This filter executes a SQL query and store the result set in the event."
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
10
10
  s.require_paths = ["lib"]
11
11
 
12
12
  # Files
13
- s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
13
+ s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
14
14
 
15
15
  # Tests
16
16
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-jdbc_streaming
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-03-27 00:00:00.000000000 Z
11
+ date: 2017-06-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -110,6 +110,7 @@ files:
110
110
  - Gemfile
111
111
  - LICENSE
112
112
  - README.md
113
+ - docs/index.asciidoc
113
114
  - lib/logstash/filters/jdbc_streaming.rb
114
115
  - lib/logstash/plugin_mixins/jdbc_streaming.rb
115
116
  - logstash-filter-jdbc_streaming.gemspec