logstash-integration-jdbc 5.4.1 → 5.4.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/docs/filter-jdbc_static.asciidoc +14 -2
- data/lib/logstash/plugin_mixins/jdbc/jdbc.rb +18 -4
- data/logstash-integration-jdbc.gemspec +1 -1
- data/spec/inputs/jdbc_spec.rb +28 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c54f1c22363c2d0637b44a2b68a98e22a1b2f07bc68927a4cc9bd6788a6fc009
|
4
|
+
data.tar.gz: 3ea76ee4fcdde152e43af6c0aab53be50dc022a8c31fce8ebac36669370a4875
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d31cef0d20064897337825bcf7ff491d3b64295c47b644d527974f458ac2c8a180d930b84b55fac6601a5f76faa10754b8975af681c4e0718ec2711e08a0d571
|
7
|
+
data.tar.gz: 6434512f2b958b976a12a7c4afe3d584066cb505c4818d8e340e363a7db4327a433785fc9ddfddc6c67eb2d719fc3f518346974760003873250895cb396d36e1
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 5.4.2
|
2
|
+
- Doc: described default_hash and tag_on_default_use interaction filter plugin [#122](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/122)
|
3
|
+
- Added new settings `statement_retry_attempts` and `statement_retry_attempts_wait_time` for retry of failed sql statement execution [#123](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/123)
|
4
|
+
|
1
5
|
## 5.4.1
|
2
6
|
- Bugfix leak which happened in creating a new Database pool for every query. The pool is now crated on registration and closed on plugin's `stop` [#119](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/119)
|
3
7
|
|
@@ -111,10 +111,14 @@ filter {
|
|
111
111
|
query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?"
|
112
112
|
prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4>
|
113
113
|
target => "user" <5>
|
114
|
+
default_hash => { <6>
|
115
|
+
firstname => nil
|
116
|
+
lastname => nil
|
117
|
+
}
|
114
118
|
}
|
115
119
|
]
|
116
120
|
# using add_field here to add & rename values to the event root
|
117
|
-
add_field => { server_name => "%{[server][0][description]}" } <
|
121
|
+
add_field => { server_name => "%{[server][0][description]}" } <7>
|
118
122
|
add_field => { user_firstname => "%{[user][0][firstname]}" }
|
119
123
|
add_field => { user_lastname => "%{[user][0][lastname]}" }
|
120
124
|
remove_field => ["server", "user"]
|
@@ -127,6 +131,13 @@ filter {
|
|
127
131
|
jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2"
|
128
132
|
}
|
129
133
|
}
|
134
|
+
|
135
|
+
output {
|
136
|
+
if "_jdbcstaticdefaultsused" in [tags] {
|
137
|
+
# Print all the not found users
|
138
|
+
stdout { }
|
139
|
+
}
|
140
|
+
}
|
130
141
|
-----
|
131
142
|
<1> Queries an external database to fetch the dataset that will be cached
|
132
143
|
locally.
|
@@ -139,7 +150,8 @@ See <<plugins-{type}s-{plugin}-object_order>>.
|
|
139
150
|
follow the positional ordering.
|
140
151
|
<5> Specifies the event field that will store the looked-up data. If the lookup
|
141
152
|
returns multiple columns, the data is stored as a JSON object within the field.
|
142
|
-
<6>
|
153
|
+
<6> When the user is not found in the database, an event is created using data from the <<plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<plugins-{type}s-{plugin}-tag_on_default_use>>.
|
154
|
+
<7> Takes data from the JSON object and stores it in top-level event fields for
|
143
155
|
easier analysis in Kibana.
|
144
156
|
|
145
157
|
Here's a full example:
|
@@ -104,6 +104,11 @@ module LogStash module PluginMixins module Jdbc
|
|
104
104
|
# Number of seconds to sleep between connection attempts
|
105
105
|
config :connection_retry_attempts_wait_time, :validate => :number, :default => 0.5
|
106
106
|
|
107
|
+
# Maximum number of times to try running statement
|
108
|
+
config :statement_retry_attempts, :validate => :number, :default => 1
|
109
|
+
# Number of seconds to sleep between statement execution
|
110
|
+
config :statement_retry_attempts_wait_time, :validate => :number, :default => 0.5
|
111
|
+
|
107
112
|
# give users the ability to force Sequel application side into using local timezone
|
108
113
|
config :plugin_timezone, :validate => ["local", "utc"], :default => "utc"
|
109
114
|
end
|
@@ -202,7 +207,10 @@ module LogStash module PluginMixins module Jdbc
|
|
202
207
|
public
|
203
208
|
def execute_statement
|
204
209
|
success = false
|
210
|
+
retry_attempts = @statement_retry_attempts
|
211
|
+
|
205
212
|
begin
|
213
|
+
retry_attempts -= 1
|
206
214
|
sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
|
207
215
|
@tracking_column_warning_sent = false
|
208
216
|
@statement_handler.perform_query(@database, @value_tracker.value) do |row|
|
@@ -210,17 +218,23 @@ module LogStash module PluginMixins module Jdbc
|
|
210
218
|
yield extract_values_from(row)
|
211
219
|
end
|
212
220
|
success = true
|
213
|
-
rescue Sequel::
|
214
|
-
Sequel::DatabaseError,
|
215
|
-
Sequel::InvalidValue,
|
216
|
-
Java::JavaSql::SQLException => e
|
221
|
+
rescue Sequel::Error, Java::JavaSql::SQLException => e
|
217
222
|
details = { exception: e.class, message: e.message }
|
218
223
|
details[:cause] = e.cause.inspect if e.cause
|
219
224
|
details[:backtrace] = e.backtrace if @logger.debug?
|
220
225
|
@logger.warn("Exception when executing JDBC query", details)
|
226
|
+
|
227
|
+
if retry_attempts == 0
|
228
|
+
@logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
|
229
|
+
else
|
230
|
+
@logger.error("Unable to execute statement. Trying again.")
|
231
|
+
sleep(@statement_retry_attempts_wait_time)
|
232
|
+
retry
|
233
|
+
end
|
221
234
|
else
|
222
235
|
@value_tracker.set_value(sql_last_value)
|
223
236
|
end
|
237
|
+
|
224
238
|
return success
|
225
239
|
end
|
226
240
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-jdbc'
|
3
|
-
s.version = '5.4.
|
3
|
+
s.version = '5.4.2'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Integration with JDBC - input and filter plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/jdbc_spec.rb
CHANGED
@@ -1352,6 +1352,34 @@ describe LogStash::Inputs::Jdbc do
|
|
1352
1352
|
expect { plugin.register }.to_not raise_error
|
1353
1353
|
plugin.stop
|
1354
1354
|
end
|
1355
|
+
|
1356
|
+
it "does retry when query execution fails" do
|
1357
|
+
mixin_settings['statement_retry_attempts'] = 2
|
1358
|
+
mixin_settings['statement_retry_attempts_wait_time'] = 0.5
|
1359
|
+
queue = Queue.new
|
1360
|
+
plugin.register
|
1361
|
+
|
1362
|
+
handler = plugin.instance_variable_get(:@statement_handler)
|
1363
|
+
allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_raise(Sequel::PoolTimeout)
|
1364
|
+
expect(plugin.logger).to receive(:error).with("Unable to execute statement. Trying again.")
|
1365
|
+
expect(plugin.logger).to receive(:error).with("Unable to execute statement. Tried 2 times.")
|
1366
|
+
|
1367
|
+
plugin.run(queue)
|
1368
|
+
plugin.stop
|
1369
|
+
end
|
1370
|
+
|
1371
|
+
it "does not retry when query execution succeeds" do
|
1372
|
+
mixin_settings['connection_retry_attempts'] = 2
|
1373
|
+
queue = Queue.new
|
1374
|
+
plugin.register
|
1375
|
+
|
1376
|
+
handler = plugin.instance_variable_get(:@statement_handler)
|
1377
|
+
allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_call_original
|
1378
|
+
expect(plugin.logger).not_to receive(:error)
|
1379
|
+
|
1380
|
+
plugin.run(queue)
|
1381
|
+
plugin.stop
|
1382
|
+
end
|
1355
1383
|
end
|
1356
1384
|
|
1357
1385
|
context "when encoding of some columns need to be changed" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-integration-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.4.
|
4
|
+
version: 5.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-05-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|