logstash-integration-jdbc 5.0.2 → 5.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +21 -1
- data/README.md +1 -1
- data/docs/filter-jdbc_static.asciidoc +2 -1
- data/docs/filter-jdbc_streaming.asciidoc +2 -1
- data/docs/input-jdbc.asciidoc +2 -1
- data/lib/logstash/filters/jdbc/basic_database.rb +1 -1
- data/lib/logstash/filters/jdbc/lookup.rb +17 -33
- data/lib/logstash/filters/jdbc_streaming.rb +2 -0
- data/lib/logstash/inputs/jdbc.rb +2 -0
- data/lib/logstash/plugin_mixins/jdbc/common.rb +66 -0
- data/lib/logstash/plugin_mixins/jdbc/jdbc.rb +19 -57
- data/lib/logstash/plugin_mixins/jdbc/statement_handler.rb +8 -1
- data/lib/logstash/plugin_mixins/jdbc_streaming.rb +2 -28
- data/logstash-integration-jdbc.gemspec +1 -1
- data/spec/filters/integration/jdbc_static_spec.rb +4 -7
- data/spec/filters/jdbc/read_only_database_spec.rb +19 -10
- data/spec/filters/jdbc/read_write_database_spec.rb +8 -6
- data/spec/inputs/integration/integ_spec.rb +18 -3
- data/spec/inputs/jdbc_spec.rb +57 -2
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 07f4d0662b489114358564f9c857f22de3dab11f0210eae7ef95ef4117b7dad7
|
4
|
+
data.tar.gz: aa469bddf21e83acee4144b3d257099de0038b221ffa4914624c0421b1d6e17d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 91f9242f4d6d425272b5737491e519fd91c6757a8e5174a519192cc7871c13ecacb6b49e4874524aff3fafaacf509bf2824feb4a733242fd08700d8af74a17d6
|
7
|
+
data.tar.gz: fa2d734620f5147ed32841b0ad75fa4ff30905ea3f5af1ff1da78204c6ea9455192d5bde197f5d860e2e658a4d6a003c42180bc4bfa8fb5999cef8c0d6da2240
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,26 @@
|
|
1
|
+
## 5.0.7
|
2
|
+
- Feat: try hard to log Java cause (chain) [#62](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/62)
|
3
|
+
|
4
|
+
This allows seeing a full trace from the JDBC driver in case of connection errors.
|
5
|
+
|
6
|
+
- Refactored Lookup used in jdbc_streaming and jdbc_static to avoid code duplication. [#59](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/59)
|
7
|
+
|
8
|
+
## 5.0.6
|
9
|
+
- DOC:Replaced plugin_header file with plugin_header-integration file. [#40](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/40)
|
10
|
+
|
11
|
+
## 5.0.5
|
12
|
+
- Fixed user sequel_opts not being passed down properly [#37](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/37)
|
13
|
+
- Refactored jdbc_streaming to share driver loading, so the fixes from the jdbc plugin also effect jdbc_streaming
|
14
|
+
|
15
|
+
## 5.0.4
|
16
|
+
- Fixed issue where JDBC Drivers that don't correctly register with Java's DriverManager fail to load (such as Sybase) [#34](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/34)
|
17
|
+
|
18
|
+
## 5.0.3
|
19
|
+
- Fixed issue where a lost connection to the database can cause errors when using prepared statements with the scheduler [#25](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/25)
|
20
|
+
|
1
21
|
## 5.0.2
|
2
22
|
- Fixed potential resource leak by ensuring scheduler is shutdown when a pipeline encounter an error during the running [#28](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/28)
|
3
|
-
|
23
|
+
|
4
24
|
## 5.0.1
|
5
25
|
- Fixed tracking_column regression with Postgresql Numeric types [#17](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/17)
|
6
26
|
- Fixed driver loading when file not accessible [#15](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/15)
|
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
Logstash Integration Plugin for JDBC, including Logstash Input and Filter Plugins
|
3
3
|
# Logstash Plugin
|
4
4
|
|
5
|
-
[![Travis Build Status](https://travis-ci.
|
5
|
+
[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-integration-jdbc.svg)](https://travis-ci.com/logstash-plugins/logstash-integration-jdbc)
|
6
6
|
|
7
7
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
8
8
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
:integration: jdbc
|
1
2
|
:plugin: jdbc_static
|
2
3
|
:type: filter
|
3
4
|
|
@@ -16,7 +17,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
|
|
16
17
|
|
17
18
|
=== Jdbc_static filter plugin
|
18
19
|
|
19
|
-
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
include::{include_path}/plugin_header-integration.asciidoc[]
|
20
21
|
|
21
22
|
==== Description
|
22
23
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
:integration: jdbc
|
1
2
|
:plugin: jdbc_streaming
|
2
3
|
:type: filter
|
3
4
|
|
@@ -16,7 +17,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
|
|
16
17
|
|
17
18
|
=== Jdbc_streaming filter plugin
|
18
19
|
|
19
|
-
include::{include_path}/plugin_header.asciidoc[]
|
20
|
+
include::{include_path}/plugin_header-integration.asciidoc[]
|
20
21
|
|
21
22
|
==== Description
|
22
23
|
|
data/docs/input-jdbc.asciidoc
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
:integration: jdbc
|
1
2
|
:plugin: jdbc
|
2
3
|
:type: input
|
3
4
|
:default_codec: plain
|
@@ -17,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
|
|
17
18
|
|
18
19
|
=== Jdbc input plugin
|
19
20
|
|
20
|
-
include::{include_path}/plugin_header.asciidoc[]
|
21
|
+
include::{include_path}/plugin_header-integration.asciidoc[]
|
21
22
|
|
22
23
|
==== Description
|
23
24
|
|
@@ -95,7 +95,7 @@ module LogStash module Filters module Jdbc
|
|
95
95
|
end
|
96
96
|
begin
|
97
97
|
db = nil
|
98
|
-
::Sequel::JDBC.load_driver(driver_class)
|
98
|
+
@options_hash[:driver] = ::Sequel::JDBC.load_driver(driver_class)
|
99
99
|
@connection_string = connection_string
|
100
100
|
if user
|
101
101
|
@options_hash[:user] = user
|
@@ -66,6 +66,7 @@ module LogStash module Filters module Jdbc
|
|
66
66
|
@prepared_statement = nil
|
67
67
|
@symbol_parameters = nil
|
68
68
|
parse_options
|
69
|
+
@load_method_ref = method(:load_data_from_local)
|
69
70
|
end
|
70
71
|
|
71
72
|
def id_used_as_target?
|
@@ -81,11 +82,7 @@ module LogStash module Filters module Jdbc
|
|
81
82
|
end
|
82
83
|
|
83
84
|
def enhance(local, event)
|
84
|
-
|
85
|
-
result = call_prepared(local, event)
|
86
|
-
else
|
87
|
-
result = fetch(local, event) # should return a LookupResult
|
88
|
-
end
|
85
|
+
result = retrieve_local_data(local, event, &@load_method_ref) # return a LookupResult
|
89
86
|
if result.failed? || result.parameters_invalid?
|
90
87
|
tag_failure(event)
|
91
88
|
end
|
@@ -112,6 +109,7 @@ module LogStash module Filters module Jdbc
|
|
112
109
|
@prepared_parameters.each_with_index { |v, i| hash[:"$p#{i}"] = v }
|
113
110
|
@prepared_param_placeholder_map = hash
|
114
111
|
@prepared_statement = local.prepare(query, hash.keys)
|
112
|
+
@load_method_ref = method(:load_data_from_prepared)
|
115
113
|
end
|
116
114
|
|
117
115
|
private
|
@@ -128,34 +126,23 @@ module LogStash module Filters module Jdbc
|
|
128
126
|
end
|
129
127
|
end
|
130
128
|
|
131
|
-
def
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
if result.parameters_invalid?
|
136
|
-
logger.warn? && logger.warn("Parameter field not found in event", :lookup_id => @id, :invalid_parameters => result.invalid_parameters)
|
137
|
-
return result
|
138
|
-
end
|
139
|
-
else
|
140
|
-
params = {}
|
129
|
+
def load_data_from_local(local, query, params, result)
|
130
|
+
local.fetch(query, params).each do |row|
|
131
|
+
stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
|
132
|
+
result.push(stringified)
|
141
133
|
end
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
rescue ::Sequel::Error => e
|
149
|
-
# all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
|
150
|
-
result.failed!
|
151
|
-
logger.warn? && logger.warn("Exception when executing Jdbc query", :lookup_id => @id, :exception => e.message, :backtrace => e.backtrace.take(8))
|
134
|
+
end
|
135
|
+
|
136
|
+
def load_data_from_prepared(_local, _query, params, result)
|
137
|
+
@prepared_statement.call(params).each do |row|
|
138
|
+
stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
|
139
|
+
result.push(stringified)
|
152
140
|
end
|
153
|
-
# if either of: no records or a Sequel exception occurs the payload is
|
154
|
-
# empty and the default can be substituted later.
|
155
|
-
result
|
156
141
|
end
|
157
142
|
|
158
|
-
|
143
|
+
# the &block is invoked with 4 arguments: local, query[String], params[Hash], result[LookupResult]
|
144
|
+
# the result is used as accumulator return variable
|
145
|
+
def retrieve_local_data(local, event, &proc)
|
159
146
|
result = LookupResult.new()
|
160
147
|
if @parameters_specified
|
161
148
|
params = prepare_parameters_from_event(event, result)
|
@@ -168,10 +155,7 @@ module LogStash module Filters module Jdbc
|
|
168
155
|
end
|
169
156
|
begin
|
170
157
|
logger.debug? && logger.debug("Executing Jdbc query", :lookup_id => @id, :statement => query, :parameters => params)
|
171
|
-
|
172
|
-
stringified = row.inject({}){|hash,(k,v)| hash[k.to_s] = v; hash} #Stringify row keys
|
173
|
-
result.push(stringified)
|
174
|
-
end
|
158
|
+
proc.call(local, query, params, result)
|
175
159
|
rescue ::Sequel::Error => e
|
176
160
|
# all sequel errors are a subclass of this, let all other standard or runtime errors bubble up
|
177
161
|
result.failed!
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/filters/base"
|
3
3
|
require "logstash/namespace"
|
4
|
+
require "logstash/plugin_mixins/jdbc/common"
|
4
5
|
require "logstash/plugin_mixins/jdbc_streaming"
|
5
6
|
require "logstash/plugin_mixins/jdbc_streaming/cache_payload"
|
6
7
|
require "logstash/plugin_mixins/jdbc_streaming/statement_handler"
|
@@ -46,6 +47,7 @@ require "lru_redux"
|
|
46
47
|
# }
|
47
48
|
#
|
48
49
|
module LogStash module Filters class JdbcStreaming < LogStash::Filters::Base
|
50
|
+
include LogStash::PluginMixins::Jdbc::Common
|
49
51
|
include LogStash::PluginMixins::JdbcStreaming
|
50
52
|
|
51
53
|
config_name "jdbc_streaming"
|
data/lib/logstash/inputs/jdbc.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/inputs/base"
|
3
3
|
require "logstash/namespace"
|
4
|
+
require "logstash/plugin_mixins/jdbc/common"
|
4
5
|
require "logstash/plugin_mixins/jdbc/jdbc"
|
5
6
|
|
6
7
|
# this require_relative returns early unless the JRuby version is between 9.2.0.0 and 9.2.8.0
|
@@ -126,6 +127,7 @@ require_relative "tzinfo_jruby_patch"
|
|
126
127
|
# ---------------------------------------------------------------------------------------------------
|
127
128
|
#
|
128
129
|
module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
130
|
+
include LogStash::PluginMixins::Jdbc::Common
|
129
131
|
include LogStash::PluginMixins::Jdbc::Jdbc
|
130
132
|
config_name "jdbc"
|
131
133
|
|
@@ -0,0 +1,66 @@
|
|
1
|
+
|
2
|
+
module LogStash module PluginMixins module Jdbc
|
3
|
+
module Common
|
4
|
+
|
5
|
+
private
|
6
|
+
|
7
|
+
def complete_sequel_opts(defaults = {})
|
8
|
+
sequel_opts = @sequel_opts.
|
9
|
+
map { |key,val| [key.is_a?(String) ? key.to_sym : key, val] }.
|
10
|
+
map { |key,val| [key, val.eql?('true') ? true : (val.eql?('false') ? false : val)] }
|
11
|
+
sequel_opts = defaults.merge Hash[sequel_opts]
|
12
|
+
sequel_opts[:user] = @jdbc_user unless @jdbc_user.nil? || @jdbc_user.empty?
|
13
|
+
sequel_opts[:password] = @jdbc_password.value unless @jdbc_password.nil?
|
14
|
+
sequel_opts[:driver] = @driver_impl # Sequel uses this as a fallback, if given URI doesn't auto-load the driver correctly
|
15
|
+
sequel_opts
|
16
|
+
end
|
17
|
+
|
18
|
+
def load_driver
|
19
|
+
return @driver_impl if @driver_impl ||= nil
|
20
|
+
|
21
|
+
require "java"
|
22
|
+
require "sequel"
|
23
|
+
require "sequel/adapters/jdbc"
|
24
|
+
|
25
|
+
load_driver_jars
|
26
|
+
begin
|
27
|
+
@driver_impl = Sequel::JDBC.load_driver(@jdbc_driver_class)
|
28
|
+
rescue Sequel::AdapterNotFound => e # Sequel::AdapterNotFound, "#{@jdbc_driver_class} not loaded"
|
29
|
+
# fix this !!!
|
30
|
+
message = if jdbc_driver_library_set?
|
31
|
+
"Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
|
32
|
+
else
|
33
|
+
":jdbc_driver_library is not set, are you sure you included " +
|
34
|
+
"the proper driver client libraries in your classpath?"
|
35
|
+
end
|
36
|
+
raise LogStash::PluginLoadingError, "#{e}. #{message}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def load_driver_jars
|
41
|
+
if jdbc_driver_library_set?
|
42
|
+
@jdbc_driver_library.split(",").each do |driver_jar|
|
43
|
+
@logger.debug("loading #{driver_jar}")
|
44
|
+
# load 'driver.jar' is different than load 'some.rb' as it only causes the file to be added to
|
45
|
+
# JRuby's class-loader lookup (class) path - won't raise a LoadError when file is not readable
|
46
|
+
unless FileTest.readable?(driver_jar)
|
47
|
+
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, " +
|
48
|
+
"file not readable (please check user and group permissions for the path)"
|
49
|
+
end
|
50
|
+
begin
|
51
|
+
require driver_jar
|
52
|
+
rescue LoadError => e
|
53
|
+
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
|
54
|
+
rescue StandardError => e
|
55
|
+
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e}"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def jdbc_driver_library_set?
|
62
|
+
!@jdbc_driver_library.nil? && !@jdbc_driver_library.empty?
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end end end
|
@@ -106,17 +106,12 @@ module LogStash module PluginMixins module Jdbc
|
|
106
106
|
|
107
107
|
private
|
108
108
|
def jdbc_connect
|
109
|
-
|
110
|
-
:user => @jdbc_user,
|
111
|
-
:password => @jdbc_password.nil? ? nil : @jdbc_password.value,
|
112
|
-
:pool_timeout => @jdbc_pool_timeout,
|
113
|
-
:keep_reference => false
|
114
|
-
}.merge(@sequel_opts)
|
109
|
+
sequel_opts = complete_sequel_opts(:pool_timeout => @jdbc_pool_timeout, :keep_reference => false)
|
115
110
|
retry_attempts = @connection_retry_attempts
|
116
111
|
loop do
|
117
112
|
retry_attempts -= 1
|
118
113
|
begin
|
119
|
-
return Sequel.connect(@jdbc_connection_string,
|
114
|
+
return Sequel.connect(@jdbc_connection_string, sequel_opts)
|
120
115
|
rescue Sequel::PoolTimeout => e
|
121
116
|
if retry_attempts <= 0
|
122
117
|
@logger.error("Failed to connect to database. #{@jdbc_pool_timeout} second timeout exceeded. Tried #{@connection_retry_attempts} times.")
|
@@ -127,64 +122,30 @@ module LogStash module PluginMixins module Jdbc
|
|
127
122
|
# rescue Java::JavaSql::SQLException, ::Sequel::Error => e
|
128
123
|
rescue ::Sequel::Error => e
|
129
124
|
if retry_attempts <= 0
|
130
|
-
|
125
|
+
log_java_exception(e.cause)
|
126
|
+
@logger.error("Unable to connect to database. Tried #{@connection_retry_attempts} times", error_details(e, trace: true))
|
131
127
|
raise e
|
132
128
|
else
|
133
|
-
@logger.error("Unable to connect to database. Trying again", :
|
129
|
+
@logger.error("Unable to connect to database. Trying again", error_details(e, trace: false))
|
134
130
|
end
|
135
131
|
end
|
136
132
|
sleep(@connection_retry_attempts_wait_time)
|
137
133
|
end
|
138
134
|
end
|
139
135
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
if @
|
144
|
-
|
145
|
-
require "sequel"
|
146
|
-
require "sequel/adapters/jdbc"
|
147
|
-
|
148
|
-
load_driver_jars
|
149
|
-
begin
|
150
|
-
Sequel::JDBC.load_driver(@jdbc_driver_class)
|
151
|
-
rescue Sequel::AdapterNotFound => e # Sequel::AdapterNotFound, "#{@jdbc_driver_class} not loaded"
|
152
|
-
# fix this !!!
|
153
|
-
message = if jdbc_driver_library_set?
|
154
|
-
"Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
|
155
|
-
else
|
156
|
-
":jdbc_driver_library is not set, are you sure you included " +
|
157
|
-
"the proper driver client libraries in your classpath?"
|
158
|
-
end
|
159
|
-
raise LogStash::PluginLoadingError, "#{e}. #{message}"
|
160
|
-
end
|
161
|
-
@drivers_loaded.make_true
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
def load_driver_jars
|
166
|
-
if jdbc_driver_library_set?
|
167
|
-
@jdbc_driver_library.split(",").each do |driver_jar|
|
168
|
-
@logger.debug("loading #{driver_jar}")
|
169
|
-
# load 'driver.jar' is different than load 'some.rb' as it only causes the file to be added to
|
170
|
-
# JRuby's class-loader lookup (class) path - won't raise a LoadError when file is not readable
|
171
|
-
unless FileTest.readable?(driver_jar)
|
172
|
-
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, " +
|
173
|
-
"file not readable (please check user and group permissions for the path)"
|
174
|
-
end
|
175
|
-
begin
|
176
|
-
require driver_jar
|
177
|
-
rescue LoadError => e
|
178
|
-
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
|
179
|
-
rescue StandardError => e
|
180
|
-
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e}"
|
181
|
-
end
|
182
|
-
end
|
183
|
-
end
|
136
|
+
def error_details(e, trace: false)
|
137
|
+
details = { :message => e.message, :exception => e.class }
|
138
|
+
details[:cause] = e.cause if e.cause
|
139
|
+
details[:backtrace] = e.backtrace if trace || @logger.debug?
|
140
|
+
details
|
184
141
|
end
|
185
142
|
|
186
|
-
def
|
187
|
-
|
143
|
+
def log_java_exception(e)
|
144
|
+
return unless e.is_a?(java.lang.Exception)
|
145
|
+
# @logger.name using the same convention as LS does
|
146
|
+
logger = self.class.name.gsub('::', '.').downcase
|
147
|
+
logger = org.apache.logging.log4j.LogManager.getLogger(logger)
|
148
|
+
logger.error('', e) # prints nested causes
|
188
149
|
end
|
189
150
|
|
190
151
|
def open_jdbc_connection
|
@@ -229,7 +190,6 @@ module LogStash module PluginMixins module Jdbc
|
|
229
190
|
public
|
230
191
|
def prepare_jdbc_connection
|
231
192
|
@connection_lock = ReentrantLock.new
|
232
|
-
@drivers_loaded = Concurrent::AtomicBoolean.new
|
233
193
|
end
|
234
194
|
|
235
195
|
public
|
@@ -260,7 +220,9 @@ module LogStash module PluginMixins module Jdbc
|
|
260
220
|
end
|
261
221
|
success = true
|
262
222
|
rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError, Java::JavaSql::SQLException => e
|
263
|
-
|
223
|
+
details = { :exception => e.message }
|
224
|
+
details[:backtrace] = e.backtrace if @logger.debug?
|
225
|
+
@logger.warn("Exception when executing JDBC query", details)
|
264
226
|
else
|
265
227
|
@value_tracker.set_value(sql_last_value)
|
266
228
|
ensure
|
@@ -97,7 +97,14 @@ module LogStash module PluginMixins module Jdbc
|
|
97
97
|
end
|
98
98
|
bind_value_sql_last_value(sql_last_value)
|
99
99
|
statement_logger.log_statement_parameters(statement, parameters, nil)
|
100
|
-
|
100
|
+
begin
|
101
|
+
db.call(name, parameters)
|
102
|
+
rescue => e
|
103
|
+
# clear the statement prepared flag - the statement may be closed by this
|
104
|
+
# time.
|
105
|
+
statement_prepared.make_false
|
106
|
+
raise e
|
107
|
+
end
|
101
108
|
end
|
102
109
|
|
103
110
|
def post_init(plugin)
|
@@ -55,37 +55,11 @@ module LogStash module PluginMixins module JdbcStreaming
|
|
55
55
|
config :jdbc_validation_timeout, :validate => :number, :default => 3600
|
56
56
|
end
|
57
57
|
|
58
|
-
private
|
59
|
-
|
60
|
-
def load_driver_jars
|
61
|
-
unless @jdbc_driver_library.nil? || @jdbc_driver_library.empty?
|
62
|
-
@jdbc_driver_library.split(",").each do |driver_jar|
|
63
|
-
begin
|
64
|
-
@logger.debug("loading #{driver_jar}")
|
65
|
-
# Use https://github.com/jruby/jruby/wiki/CallingJavaFromJRuby#from-jar-files to make classes from jar
|
66
|
-
# available
|
67
|
-
require driver_jar
|
68
|
-
rescue LoadError => e
|
69
|
-
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
58
|
public
|
76
59
|
def prepare_jdbc_connection
|
77
|
-
|
78
|
-
require "sequel/adapters/jdbc"
|
79
|
-
require "java"
|
80
|
-
|
81
|
-
load_driver_jars
|
82
|
-
|
83
|
-
@sequel_opts_symbols = @sequel_opts.inject({}) {|hash, (k,v)| hash[k.to_sym] = v; hash}
|
84
|
-
@sequel_opts_symbols[:user] = @jdbc_user unless @jdbc_user.nil? || @jdbc_user.empty?
|
85
|
-
@sequel_opts_symbols[:password] = @jdbc_password.value unless @jdbc_password.nil?
|
60
|
+
load_driver
|
86
61
|
|
87
|
-
Sequel
|
88
|
-
@database = Sequel.connect(@jdbc_connection_string, @sequel_opts_symbols)
|
62
|
+
@database = Sequel.connect(@jdbc_connection_string, complete_sequel_opts)
|
89
63
|
if @jdbc_validate_connection
|
90
64
|
@database.extension(:connection_validator)
|
91
65
|
@database.pool.connection_validation_timeout = @jdbc_validation_timeout
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-jdbc'
|
3
|
-
s.version = '5.0.
|
3
|
+
s.version = '5.0.7'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Integration with JDBC - input and filter plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -4,7 +4,6 @@ require "logstash/filters/jdbc_static"
|
|
4
4
|
require "sequel"
|
5
5
|
require "sequel/adapters/jdbc"
|
6
6
|
require "stud/temporary"
|
7
|
-
require "timecop"
|
8
7
|
|
9
8
|
module LogStash module Filters
|
10
9
|
describe JdbcStatic, :integration => true do
|
@@ -60,7 +59,7 @@ module LogStash module Filters
|
|
60
59
|
|
61
60
|
let(:plugin) { JdbcStatic.new(settings) }
|
62
61
|
|
63
|
-
let(:event)
|
62
|
+
let(:event) { ::LogStash::Event.new("message" => "some text", "ip" => ipaddr) }
|
64
63
|
|
65
64
|
let(:ipaddr) { ".3.1.1" }
|
66
65
|
|
@@ -136,18 +135,16 @@ module LogStash module Filters
|
|
136
135
|
describe "scheduled operation" do
|
137
136
|
context "given a loader_schedule" do
|
138
137
|
it "should properly schedule" do
|
139
|
-
settings["loader_schedule"] = "*/
|
140
|
-
Timecop.travel(Time.now.utc - 3600)
|
141
|
-
Timecop.scale(60)
|
138
|
+
settings["loader_schedule"] = "*/3 * * * * * UTC"
|
142
139
|
static_filter = JdbcStatic.new(settings)
|
143
140
|
runner = Thread.new(static_filter) do |filter|
|
144
141
|
filter.register
|
145
142
|
end
|
146
|
-
|
143
|
+
runner.join(4)
|
144
|
+
sleep 4
|
147
145
|
static_filter.filter(event)
|
148
146
|
expect(static_filter.loader_runner.reload_count).to be > 1
|
149
147
|
static_filter.close
|
150
|
-
Timecop.return
|
151
148
|
expect(event.get("server")).to eq([{"ip"=>"10.3.1.1", "name"=>"mv-server-1", "location"=>"MV-9-6-4"}])
|
152
149
|
end
|
153
150
|
end
|
@@ -10,28 +10,37 @@ module LogStash module Filters module Jdbc
|
|
10
10
|
let(:driver_class) { "org.apache.derby.jdbc.EmbeddedDriver" }
|
11
11
|
let(:driver_library) { nil }
|
12
12
|
subject(:read_only_db) { described_class.create(connection_string, driver_class, driver_library) }
|
13
|
+
let(:stub_driver_class) { double(driver_class).as_null_object }
|
13
14
|
|
14
15
|
describe "basic operations" do
|
15
16
|
describe "initializing" do
|
17
|
+
before(:each) do
|
18
|
+
allow(Sequel::JDBC).to receive(:load_driver).and_return(stub_driver_class)
|
19
|
+
end
|
20
|
+
|
16
21
|
it "tests the connection with defaults" do
|
17
22
|
expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
|
18
|
-
expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true})
|
23
|
+
expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true, :driver => stub_driver_class})
|
19
24
|
expect(read_only_db.empty_record_set).to eq([])
|
20
25
|
end
|
21
26
|
|
22
|
-
|
23
|
-
|
24
|
-
user
|
25
|
-
password
|
26
|
-
|
27
|
-
|
28
|
-
|
27
|
+
context 'with fully-specified arguments' do
|
28
|
+
let(:connection_string) { "a connection string" }
|
29
|
+
let(:user) { "a user" }
|
30
|
+
let(:password) { Util::Password.new("secret") }
|
31
|
+
let(:driver_class) { "a driver class" }
|
32
|
+
|
33
|
+
it "tests the connection" do
|
34
|
+
expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
|
35
|
+
expect(Sequel).to receive(:connect).once.with(connection_string, {:user => user, :password => password.value, :test => true, :driver => stub_driver_class}).and_return(db)
|
36
|
+
described_class.create(connection_string, driver_class, nil, user, password)
|
37
|
+
end
|
29
38
|
end
|
30
39
|
|
31
40
|
it "connects with defaults" do
|
32
41
|
expect(Sequel::JDBC).to receive(:load_driver).once.with(driver_class)
|
33
|
-
expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true}).and_return(db)
|
34
|
-
expect(Sequel).to receive(:connect).once.with(connection_string, {}).and_return(db)
|
42
|
+
expect(Sequel).to receive(:connect).once.with(connection_string, {:test => true, :driver => stub_driver_class}).and_return(db)
|
43
|
+
expect(Sequel).to receive(:connect).once.with(connection_string, {:driver => stub_driver_class}).and_return(db)
|
35
44
|
expect(read_only_db.connected?).to be_falsey
|
36
45
|
read_only_db.connect("a caller specific error message")
|
37
46
|
expect(read_only_db.connected?).to be_truthy
|
@@ -16,10 +16,11 @@ module LogStash module Filters module Jdbc
|
|
16
16
|
describe "basic operations" do
|
17
17
|
context "connecting to a db" do
|
18
18
|
it "connects with defaults" do
|
19
|
-
|
19
|
+
stub_driver_class = double('org.apache.derby.jdbc.EmbeddedDriver').as_null_object
|
20
|
+
expect(::Sequel::JDBC).to receive(:load_driver).once.with("org.apache.derby.jdbc.EmbeddedDriver").and_return(stub_driver_class)
|
20
21
|
# two calls to connect because ReadWriteDatabase does verify_connection and connect
|
21
|
-
expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {:test => true}).and_return(db)
|
22
|
-
expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {}).and_return(db)
|
22
|
+
expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {:driver => stub_driver_class, :test => true}).and_return(db)
|
23
|
+
expect(::Sequel).to receive(:connect).once.with(connection_string_regex, {:driver => stub_driver_class}).and_return(db)
|
23
24
|
expect(read_write_db.empty_record_set).to eq([])
|
24
25
|
end
|
25
26
|
|
@@ -27,9 +28,10 @@ module LogStash module Filters module Jdbc
|
|
27
28
|
connection_str = "a connection string"
|
28
29
|
user = "a user"
|
29
30
|
password = Util::Password.new("secret")
|
30
|
-
|
31
|
-
expect(::Sequel).to receive(:
|
32
|
-
expect(::Sequel).to receive(:connect).once.with(connection_str, {:user => user, :password => password.value}).and_return(db)
|
31
|
+
stub_driver_class = double('com.example.Driver')
|
32
|
+
expect(::Sequel::JDBC).to receive(:load_driver).once.with("a driver class").and_return(stub_driver_class)
|
33
|
+
expect(::Sequel).to receive(:connect).once.with(connection_str, {:driver => stub_driver_class, :user => user, :password => password.value, :test => true}).and_return(db)
|
34
|
+
expect(::Sequel).to receive(:connect).once.with(connection_str, {:driver => stub_driver_class, :user => user, :password => password.value}).and_return(db)
|
33
35
|
described_class.create(connection_str, "a driver class", nil, user, password)
|
34
36
|
end
|
35
37
|
end
|
@@ -3,7 +3,6 @@ require "logstash/inputs/jdbc"
|
|
3
3
|
require "sequel"
|
4
4
|
require "sequel/adapters/jdbc"
|
5
5
|
|
6
|
-
# This test requires: Firebird installed to Mac OSX, it uses the built-in example database `employee`
|
7
6
|
|
8
7
|
describe LogStash::Inputs::Jdbc, :integration => true do
|
9
8
|
# This is a necessary change test-wide to guarantee that no local timezone
|
@@ -46,7 +45,7 @@ describe LogStash::Inputs::Jdbc, :integration => true do
|
|
46
45
|
|
47
46
|
context "when supplying a non-existent library" do
|
48
47
|
let(:settings) do
|
49
|
-
super.merge(
|
48
|
+
super().merge(
|
50
49
|
"jdbc_driver_library" => "/no/path/to/postgresql.jar"
|
51
50
|
)
|
52
51
|
end
|
@@ -62,13 +61,29 @@ describe LogStash::Inputs::Jdbc, :integration => true do
|
|
62
61
|
|
63
62
|
context "when connecting to a non-existent server" do
|
64
63
|
let(:settings) do
|
65
|
-
super.merge(
|
64
|
+
super().merge(
|
66
65
|
"jdbc_connection_string" => "jdbc:postgresql://localhost:65000/somedb"
|
67
66
|
)
|
68
67
|
end
|
69
68
|
|
70
69
|
it "should not register correctly" do
|
71
70
|
plugin.register
|
71
|
+
allow( plugin ).to receive(:log_java_exception)
|
72
|
+
q = Queue.new
|
73
|
+
expect do
|
74
|
+
plugin.run(q)
|
75
|
+
end.to raise_error(::Sequel::DatabaseConnectionError)
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should log (native) Java driver error" do
|
79
|
+
plugin.register
|
80
|
+
expect( org.apache.logging.log4j.LogManager ).to receive(:getLogger).and_wrap_original do |m, *args|
|
81
|
+
logger = m.call(*args)
|
82
|
+
expect( logger ).to receive(:error) do |_, e|
|
83
|
+
expect( e ).to be_a org.postgresql.util.PSQLException
|
84
|
+
end.and_call_original
|
85
|
+
logger
|
86
|
+
end
|
72
87
|
q = Queue.new
|
73
88
|
expect do
|
74
89
|
plugin.run(q)
|
data/spec/inputs/jdbc_spec.rb
CHANGED
@@ -13,9 +13,13 @@ require "date"
|
|
13
13
|
# We do not need to set TZ env var anymore because we can have 'Sequel.application_timezone' set to utc by default now.
|
14
14
|
|
15
15
|
describe LogStash::Inputs::Jdbc do
|
16
|
+
let(:connection_string) { "jdbc:derby:memory:testdb;create=true" }
|
16
17
|
let(:mixin_settings) do
|
17
|
-
{
|
18
|
-
|
18
|
+
{
|
19
|
+
"jdbc_user" => ENV['USER'],
|
20
|
+
"jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
|
21
|
+
"jdbc_connection_string" => connection_string
|
22
|
+
}
|
19
23
|
end
|
20
24
|
let(:settings) { {} }
|
21
25
|
let(:plugin) { LogStash::Inputs::Jdbc.new(mixin_settings.merge(settings)) }
|
@@ -112,6 +116,57 @@ describe LogStash::Inputs::Jdbc do
|
|
112
116
|
end
|
113
117
|
end
|
114
118
|
|
119
|
+
context "when sequel opts has user option" do
|
120
|
+
let(:settings) do
|
121
|
+
{
|
122
|
+
"jdbc_user" => 'system',
|
123
|
+
"statement" => "SELECT * from test_table",
|
124
|
+
"sequel_opts" => { "user" => 'from-opts' }
|
125
|
+
}
|
126
|
+
end
|
127
|
+
|
128
|
+
before do
|
129
|
+
plugin.register
|
130
|
+
end
|
131
|
+
|
132
|
+
after do
|
133
|
+
plugin.stop
|
134
|
+
end
|
135
|
+
|
136
|
+
it "should honor set jdbc-user when connecting" do
|
137
|
+
expect( Sequel ).to receive(:connect).with connection_string, hash_including(:user=>"system")
|
138
|
+
plugin.send(:jdbc_connect)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
context "with sequel opts" do
|
143
|
+
let(:settings) do
|
144
|
+
{
|
145
|
+
"jdbc_user" => 'system',
|
146
|
+
"statement" => "SELECT * from test_table",
|
147
|
+
"sequel_opts" => {
|
148
|
+
"truthy" => 'true',
|
149
|
+
"falsey" => 'false',
|
150
|
+
"foo" => 'bar',
|
151
|
+
"jdbc_properties" => { "some" => 'true' }
|
152
|
+
}
|
153
|
+
}
|
154
|
+
end
|
155
|
+
|
156
|
+
before do
|
157
|
+
plugin.register
|
158
|
+
end
|
159
|
+
|
160
|
+
after do
|
161
|
+
plugin.stop
|
162
|
+
end
|
163
|
+
|
164
|
+
it "should symbolize keys" do
|
165
|
+
expect( Sequel ).to receive(:connect).with connection_string,
|
166
|
+
hash_including(:truthy => true, :falsey => false, :foo => 'bar', :jdbc_properties => { 'some' => 'true' })
|
167
|
+
plugin.send(:jdbc_connect)
|
168
|
+
end
|
169
|
+
end
|
115
170
|
|
116
171
|
context "when neither statement and statement_filepath arguments are passed" do
|
117
172
|
it "should fail to register" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-integration-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.0.
|
4
|
+
version: 5.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-04-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -234,6 +234,7 @@ files:
|
|
234
234
|
- lib/logstash/inputs/jdbc.rb
|
235
235
|
- lib/logstash/inputs/tzinfo_jruby_patch.rb
|
236
236
|
- lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb
|
237
|
+
- lib/logstash/plugin_mixins/jdbc/common.rb
|
237
238
|
- lib/logstash/plugin_mixins/jdbc/jdbc.rb
|
238
239
|
- lib/logstash/plugin_mixins/jdbc/statement_handler.rb
|
239
240
|
- lib/logstash/plugin_mixins/jdbc/value_tracking.rb
|