logstash-integration-jdbc 5.4.1 → 5.4.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 71bdfca5d1e5e86b7d86b5c49caf237cfc882e31dab78d63e169241e1c96814d
4
- data.tar.gz: 5c07e44042c4a6e6e29b8ecc7fb7631147100a27777c6fdd2a22015ac7fd3b4e
3
+ metadata.gz: c54f1c22363c2d0637b44a2b68a98e22a1b2f07bc68927a4cc9bd6788a6fc009
4
+ data.tar.gz: 3ea76ee4fcdde152e43af6c0aab53be50dc022a8c31fce8ebac36669370a4875
5
5
  SHA512:
6
- metadata.gz: 53e45e47383b15f559e58969e96c0217c43ceb5ba3528db5d6ce3bad10628da6e4c14442aeb5a8795e17b667ac558e5b1377134bb0ecc5e60a375e6ebe106b2c
7
- data.tar.gz: d33a2cdea75e56fbe338e175276679e2910b4023c956bc8df7d0795f58140b39da1b9167d668c7665e27b2577c086409b3e390b2a3f8d86c4f9a3e3cdef343e3
6
+ metadata.gz: d31cef0d20064897337825bcf7ff491d3b64295c47b644d527974f458ac2c8a180d930b84b55fac6601a5f76faa10754b8975af681c4e0718ec2711e08a0d571
7
+ data.tar.gz: 6434512f2b958b976a12a7c4afe3d584066cb505c4818d8e340e363a7db4327a433785fc9ddfddc6c67eb2d719fc3f518346974760003873250895cb396d36e1
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 5.4.2
2
+ - Doc: described default_hash and tag_on_default_use interaction filter plugin [#122](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/122)
3
+ - Added new settings `statement_retry_attempts` and `statement_retry_attempts_wait_time` for retry of failed sql statement execution [#123](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/123)
4
+
1
5
  ## 5.4.1
2
6
  - Bugfix leak which happened in creating a new Database pool for every query. The pool is now crated on registration and closed on plugin's `stop` [#119](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/119)
3
7
 
@@ -111,10 +111,14 @@ filter {
111
111
  query => "SELECT firstname, lastname FROM users WHERE userid = ? AND country = ?"
112
112
  prepared_parameters => ["[loggedin_userid]", "[user_nation]"] <4>
113
113
  target => "user" <5>
114
+ default_hash => { <6>
115
+ firstname => nil
116
+ lastname => nil
117
+ }
114
118
  }
115
119
  ]
116
120
  # using add_field here to add & rename values to the event root
117
- add_field => { server_name => "%{[server][0][description]}" } <6>
121
+ add_field => { server_name => "%{[server][0][description]}" } <7>
118
122
  add_field => { user_firstname => "%{[user][0][firstname]}" }
119
123
  add_field => { user_lastname => "%{[user][0][lastname]}" }
120
124
  remove_field => ["server", "user"]
@@ -127,6 +131,13 @@ filter {
127
131
  jdbc_connection_string => "jdbc:postgresql://remotedb:5432/ls_test_2"
128
132
  }
129
133
  }
134
+
135
+ output {
136
+ if "_jdbcstaticdefaultsused" in [tags] {
137
+ # Print all the not found users
138
+ stdout { }
139
+ }
140
+ }
130
141
  -----
131
142
  <1> Queries an external database to fetch the dataset that will be cached
132
143
  locally.
@@ -139,7 +150,8 @@ See <<plugins-{type}s-{plugin}-object_order>>.
139
150
  follow the positional ordering.
140
151
  <5> Specifies the event field that will store the looked-up data. If the lookup
141
152
  returns multiple columns, the data is stored as a JSON object within the field.
142
- <6> Takes data from the JSON object and stores it in top-level event fields for
153
+ <6> When the user is not found in the database, an event is created using data from the <<plugins-{type}s-{plugin}-local_lookups>> `default hash` setting, and the event is tagged with the list set in <<plugins-{type}s-{plugin}-tag_on_default_use>>.
154
+ <7> Takes data from the JSON object and stores it in top-level event fields for
143
155
  easier analysis in Kibana.
144
156
 
145
157
  Here's a full example:
@@ -104,6 +104,11 @@ module LogStash module PluginMixins module Jdbc
104
104
  # Number of seconds to sleep between connection attempts
105
105
  config :connection_retry_attempts_wait_time, :validate => :number, :default => 0.5
106
106
 
107
+ # Maximum number of times to try running statement
108
+ config :statement_retry_attempts, :validate => :number, :default => 1
109
+ # Number of seconds to sleep between statement execution
110
+ config :statement_retry_attempts_wait_time, :validate => :number, :default => 0.5
111
+
107
112
  # give users the ability to force Sequel application side into using local timezone
108
113
  config :plugin_timezone, :validate => ["local", "utc"], :default => "utc"
109
114
  end
@@ -202,7 +207,10 @@ module LogStash module PluginMixins module Jdbc
202
207
  public
203
208
  def execute_statement
204
209
  success = false
210
+ retry_attempts = @statement_retry_attempts
211
+
205
212
  begin
213
+ retry_attempts -= 1
206
214
  sql_last_value = @use_column_value ? @value_tracker.value : Time.now.utc
207
215
  @tracking_column_warning_sent = false
208
216
  @statement_handler.perform_query(@database, @value_tracker.value) do |row|
@@ -210,17 +218,23 @@ module LogStash module PluginMixins module Jdbc
210
218
  yield extract_values_from(row)
211
219
  end
212
220
  success = true
213
- rescue Sequel::DatabaseConnectionError,
214
- Sequel::DatabaseError,
215
- Sequel::InvalidValue,
216
- Java::JavaSql::SQLException => e
221
+ rescue Sequel::Error, Java::JavaSql::SQLException => e
217
222
  details = { exception: e.class, message: e.message }
218
223
  details[:cause] = e.cause.inspect if e.cause
219
224
  details[:backtrace] = e.backtrace if @logger.debug?
220
225
  @logger.warn("Exception when executing JDBC query", details)
226
+
227
+ if retry_attempts == 0
228
+ @logger.error("Unable to execute statement. Tried #{@statement_retry_attempts} times.")
229
+ else
230
+ @logger.error("Unable to execute statement. Trying again.")
231
+ sleep(@statement_retry_attempts_wait_time)
232
+ retry
233
+ end
221
234
  else
222
235
  @value_tracker.set_value(sql_last_value)
223
236
  end
237
+
224
238
  return success
225
239
  end
226
240
 
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-integration-jdbc'
3
- s.version = '5.4.1'
3
+ s.version = '5.4.2'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Integration with JDBC - input and filter plugins"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -1352,6 +1352,34 @@ describe LogStash::Inputs::Jdbc do
1352
1352
  expect { plugin.register }.to_not raise_error
1353
1353
  plugin.stop
1354
1354
  end
1355
+
1356
+ it "does retry when query execution fails" do
1357
+ mixin_settings['statement_retry_attempts'] = 2
1358
+ mixin_settings['statement_retry_attempts_wait_time'] = 0.5
1359
+ queue = Queue.new
1360
+ plugin.register
1361
+
1362
+ handler = plugin.instance_variable_get(:@statement_handler)
1363
+ allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_raise(Sequel::PoolTimeout)
1364
+ expect(plugin.logger).to receive(:error).with("Unable to execute statement. Trying again.")
1365
+ expect(plugin.logger).to receive(:error).with("Unable to execute statement. Tried 2 times.")
1366
+
1367
+ plugin.run(queue)
1368
+ plugin.stop
1369
+ end
1370
+
1371
+ it "does not retry when query execution succeeds" do
1372
+ mixin_settings['connection_retry_attempts'] = 2
1373
+ queue = Queue.new
1374
+ plugin.register
1375
+
1376
+ handler = plugin.instance_variable_get(:@statement_handler)
1377
+ allow(handler).to receive(:perform_query).with(instance_of(Sequel::JDBC::Database), instance_of(Time)).and_call_original
1378
+ expect(plugin.logger).not_to receive(:error)
1379
+
1380
+ plugin.run(queue)
1381
+ plugin.stop
1382
+ end
1355
1383
  end
1356
1384
 
1357
1385
  context "when encoding of some columns need to be changed" do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-integration-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.4.1
4
+ version: 5.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-11-04 00:00:00.000000000 Z
11
+ date: 2023-05-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement