logstash-integration-jdbc 5.0.4 → 5.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/filters/jdbc_streaming.rb +2 -0
- data/lib/logstash/inputs/jdbc.rb +2 -0
- data/lib/logstash/plugin_mixins/jdbc/common.rb +66 -0
- data/lib/logstash/plugin_mixins/jdbc/jdbc.rb +3 -60
- data/lib/logstash/plugin_mixins/jdbc_streaming.rb +2 -28
- data/logstash-integration-jdbc.gemspec +1 -1
- data/spec/inputs/jdbc_spec.rb +57 -2
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 39722eec0f50423361c7a332054ffeaab4dde768e0a84d45535d10dc1661fc55
|
4
|
+
data.tar.gz: cd1f3ed6d68671687d921d4d8e88a8821d9536777f2f5d543954d2848f21340a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 78b8c924c351b1e80b1456b019c2e7e9fb7305410c748c15f05bc6088f1d9a9d4e0c09cfecaa71804bcad2e33b81a5610ef859023d1bdf26cce3f92db50e6c80
|
7
|
+
data.tar.gz: 593d2160fc67da503c7f5db0ff42f1c33ed65b9316d398a86c402504c149b266f6e2b1e8fbe2ff965e193341753e23d7d16585597906bb6dadb8b1356e41cb8a
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 5.0.5
|
2
|
+
- Fixed user sequel_opts not being passed down properly [#37](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/37)
|
3
|
+
- Refactored jdbc_streaming to share driver loading, so the fixes from the jdbc plugin also effect jdbc_streaming
|
4
|
+
|
1
5
|
## 5.0.4
|
2
6
|
- Fixed issue where JDBC Drivers that don't correctly register with Java's DriverManager fail to load (such as Sybase) [#34](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/34)
|
3
7
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/filters/base"
|
3
3
|
require "logstash/namespace"
|
4
|
+
require "logstash/plugin_mixins/jdbc/common"
|
4
5
|
require "logstash/plugin_mixins/jdbc_streaming"
|
5
6
|
require "logstash/plugin_mixins/jdbc_streaming/cache_payload"
|
6
7
|
require "logstash/plugin_mixins/jdbc_streaming/statement_handler"
|
@@ -46,6 +47,7 @@ require "lru_redux"
|
|
46
47
|
# }
|
47
48
|
#
|
48
49
|
module LogStash module Filters class JdbcStreaming < LogStash::Filters::Base
|
50
|
+
include LogStash::PluginMixins::Jdbc::Common
|
49
51
|
include LogStash::PluginMixins::JdbcStreaming
|
50
52
|
|
51
53
|
config_name "jdbc_streaming"
|
data/lib/logstash/inputs/jdbc.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/inputs/base"
|
3
3
|
require "logstash/namespace"
|
4
|
+
require "logstash/plugin_mixins/jdbc/common"
|
4
5
|
require "logstash/plugin_mixins/jdbc/jdbc"
|
5
6
|
|
6
7
|
# this require_relative returns early unless the JRuby version is between 9.2.0.0 and 9.2.8.0
|
@@ -126,6 +127,7 @@ require_relative "tzinfo_jruby_patch"
|
|
126
127
|
# ---------------------------------------------------------------------------------------------------
|
127
128
|
#
|
128
129
|
module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
|
130
|
+
include LogStash::PluginMixins::Jdbc::Common
|
129
131
|
include LogStash::PluginMixins::Jdbc::Jdbc
|
130
132
|
config_name "jdbc"
|
131
133
|
|
@@ -0,0 +1,66 @@
|
|
1
|
+
|
2
|
+
module LogStash module PluginMixins module Jdbc
|
3
|
+
module Common
|
4
|
+
|
5
|
+
private
|
6
|
+
|
7
|
+
def complete_sequel_opts(defaults = {})
|
8
|
+
sequel_opts = @sequel_opts.
|
9
|
+
map { |key,val| [key.is_a?(String) ? key.to_sym : key, val] }.
|
10
|
+
map { |key,val| [key, val.eql?('true') ? true : (val.eql?('false') ? false : val)] }
|
11
|
+
sequel_opts = defaults.merge Hash[sequel_opts]
|
12
|
+
sequel_opts[:user] = @jdbc_user unless @jdbc_user.nil? || @jdbc_user.empty?
|
13
|
+
sequel_opts[:password] = @jdbc_password.value unless @jdbc_password.nil?
|
14
|
+
sequel_opts[:driver] = @driver_impl # Sequel uses this as a fallback, if given URI doesn't auto-load the driver correctly
|
15
|
+
sequel_opts
|
16
|
+
end
|
17
|
+
|
18
|
+
def load_driver
|
19
|
+
return @driver_impl if @driver_impl ||= nil
|
20
|
+
|
21
|
+
require "java"
|
22
|
+
require "sequel"
|
23
|
+
require "sequel/adapters/jdbc"
|
24
|
+
|
25
|
+
load_driver_jars
|
26
|
+
begin
|
27
|
+
@driver_impl = Sequel::JDBC.load_driver(@jdbc_driver_class)
|
28
|
+
rescue Sequel::AdapterNotFound => e # Sequel::AdapterNotFound, "#{@jdbc_driver_class} not loaded"
|
29
|
+
# fix this !!!
|
30
|
+
message = if jdbc_driver_library_set?
|
31
|
+
"Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
|
32
|
+
else
|
33
|
+
":jdbc_driver_library is not set, are you sure you included " +
|
34
|
+
"the proper driver client libraries in your classpath?"
|
35
|
+
end
|
36
|
+
raise LogStash::PluginLoadingError, "#{e}. #{message}"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def load_driver_jars
|
41
|
+
if jdbc_driver_library_set?
|
42
|
+
@jdbc_driver_library.split(",").each do |driver_jar|
|
43
|
+
@logger.debug("loading #{driver_jar}")
|
44
|
+
# load 'driver.jar' is different than load 'some.rb' as it only causes the file to be added to
|
45
|
+
# JRuby's class-loader lookup (class) path - won't raise a LoadError when file is not readable
|
46
|
+
unless FileTest.readable?(driver_jar)
|
47
|
+
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, " +
|
48
|
+
"file not readable (please check user and group permissions for the path)"
|
49
|
+
end
|
50
|
+
begin
|
51
|
+
require driver_jar
|
52
|
+
rescue LoadError => e
|
53
|
+
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
|
54
|
+
rescue StandardError => e
|
55
|
+
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e}"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def jdbc_driver_library_set?
|
62
|
+
!@jdbc_driver_library.nil? && !@jdbc_driver_library.empty?
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
end end end
|
@@ -106,18 +106,12 @@ module LogStash module PluginMixins module Jdbc
|
|
106
106
|
|
107
107
|
private
|
108
108
|
def jdbc_connect
|
109
|
-
|
110
|
-
:user => @jdbc_user,
|
111
|
-
:password => @jdbc_password.nil? ? nil : @jdbc_password.value,
|
112
|
-
:pool_timeout => @jdbc_pool_timeout,
|
113
|
-
:driver => @driver_impl, # Sequel uses this as a fallback, if given URI doesn't auto-load the driver correctly
|
114
|
-
:keep_reference => false
|
115
|
-
}.merge(@sequel_opts)
|
109
|
+
sequel_opts = complete_sequel_opts(:pool_timeout => @jdbc_pool_timeout, :keep_reference => false)
|
116
110
|
retry_attempts = @connection_retry_attempts
|
117
111
|
loop do
|
118
112
|
retry_attempts -= 1
|
119
113
|
begin
|
120
|
-
return Sequel.connect(@jdbc_connection_string,
|
114
|
+
return Sequel.connect(@jdbc_connection_string, sequel_opts)
|
121
115
|
rescue Sequel::PoolTimeout => e
|
122
116
|
if retry_attempts <= 0
|
123
117
|
@logger.error("Failed to connect to database. #{@jdbc_pool_timeout} second timeout exceeded. Tried #{@connection_retry_attempts} times.")
|
@@ -128,7 +122,7 @@ module LogStash module PluginMixins module Jdbc
|
|
128
122
|
# rescue Java::JavaSql::SQLException, ::Sequel::Error => e
|
129
123
|
rescue ::Sequel::Error => e
|
130
124
|
if retry_attempts <= 0
|
131
|
-
@logger.error("Unable to connect to database. Tried #{@connection_retry_attempts} times", :error_message => e.message
|
125
|
+
@logger.error("Unable to connect to database. Tried #{@connection_retry_attempts} times", :error_message => e.message)
|
132
126
|
raise e
|
133
127
|
else
|
134
128
|
@logger.error("Unable to connect to database. Trying again", :error_message => e.message)
|
@@ -138,56 +132,6 @@ module LogStash module PluginMixins module Jdbc
|
|
138
132
|
end
|
139
133
|
end
|
140
134
|
|
141
|
-
private
|
142
|
-
|
143
|
-
def load_driver
|
144
|
-
if @drivers_loaded.false?
|
145
|
-
require "java"
|
146
|
-
require "sequel"
|
147
|
-
require "sequel/adapters/jdbc"
|
148
|
-
|
149
|
-
load_driver_jars
|
150
|
-
begin
|
151
|
-
@driver_impl = Sequel::JDBC.load_driver(@jdbc_driver_class)
|
152
|
-
rescue Sequel::AdapterNotFound => e # Sequel::AdapterNotFound, "#{@jdbc_driver_class} not loaded"
|
153
|
-
# fix this !!!
|
154
|
-
message = if jdbc_driver_library_set?
|
155
|
-
"Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
|
156
|
-
else
|
157
|
-
":jdbc_driver_library is not set, are you sure you included " +
|
158
|
-
"the proper driver client libraries in your classpath?"
|
159
|
-
end
|
160
|
-
raise LogStash::PluginLoadingError, "#{e}. #{message}"
|
161
|
-
end
|
162
|
-
@drivers_loaded.make_true
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
|
-
def load_driver_jars
|
167
|
-
if jdbc_driver_library_set?
|
168
|
-
@jdbc_driver_library.split(",").each do |driver_jar|
|
169
|
-
@logger.debug("loading #{driver_jar}")
|
170
|
-
# load 'driver.jar' is different than load 'some.rb' as it only causes the file to be added to
|
171
|
-
# JRuby's class-loader lookup (class) path - won't raise a LoadError when file is not readable
|
172
|
-
unless FileTest.readable?(driver_jar)
|
173
|
-
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, " +
|
174
|
-
"file not readable (please check user and group permissions for the path)"
|
175
|
-
end
|
176
|
-
begin
|
177
|
-
require driver_jar
|
178
|
-
rescue LoadError => e
|
179
|
-
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
|
180
|
-
rescue StandardError => e
|
181
|
-
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e}"
|
182
|
-
end
|
183
|
-
end
|
184
|
-
end
|
185
|
-
end
|
186
|
-
|
187
|
-
def jdbc_driver_library_set?
|
188
|
-
!@jdbc_driver_library.nil? && !@jdbc_driver_library.empty?
|
189
|
-
end
|
190
|
-
|
191
135
|
def open_jdbc_connection
|
192
136
|
# at this point driver is already loaded
|
193
137
|
Sequel.application_timezone = @plugin_timezone.to_sym
|
@@ -230,7 +174,6 @@ module LogStash module PluginMixins module Jdbc
|
|
230
174
|
public
|
231
175
|
def prepare_jdbc_connection
|
232
176
|
@connection_lock = ReentrantLock.new
|
233
|
-
@drivers_loaded = Concurrent::AtomicBoolean.new
|
234
177
|
end
|
235
178
|
|
236
179
|
public
|
@@ -55,37 +55,11 @@ module LogStash module PluginMixins module JdbcStreaming
|
|
55
55
|
config :jdbc_validation_timeout, :validate => :number, :default => 3600
|
56
56
|
end
|
57
57
|
|
58
|
-
private
|
59
|
-
|
60
|
-
def load_driver_jars
|
61
|
-
unless @jdbc_driver_library.nil? || @jdbc_driver_library.empty?
|
62
|
-
@jdbc_driver_library.split(",").each do |driver_jar|
|
63
|
-
begin
|
64
|
-
@logger.debug("loading #{driver_jar}")
|
65
|
-
# Use https://github.com/jruby/jruby/wiki/CallingJavaFromJRuby#from-jar-files to make classes from jar
|
66
|
-
# available
|
67
|
-
require driver_jar
|
68
|
-
rescue LoadError => e
|
69
|
-
raise LogStash::PluginLoadingError, "unable to load #{driver_jar} from :jdbc_driver_library, #{e.message}"
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
58
|
public
|
76
59
|
def prepare_jdbc_connection
|
77
|
-
|
78
|
-
require "sequel/adapters/jdbc"
|
79
|
-
require "java"
|
80
|
-
|
81
|
-
load_driver_jars
|
82
|
-
|
83
|
-
@sequel_opts_symbols = @sequel_opts.inject({}) {|hash, (k,v)| hash[k.to_sym] = v; hash}
|
84
|
-
@sequel_opts_symbols[:user] = @jdbc_user unless @jdbc_user.nil? || @jdbc_user.empty?
|
85
|
-
@sequel_opts_symbols[:password] = @jdbc_password.value unless @jdbc_password.nil?
|
60
|
+
load_driver
|
86
61
|
|
87
|
-
@
|
88
|
-
@database = Sequel.connect(@jdbc_connection_string, @sequel_opts_symbols)
|
62
|
+
@database = Sequel.connect(@jdbc_connection_string, complete_sequel_opts)
|
89
63
|
if @jdbc_validate_connection
|
90
64
|
@database.extension(:connection_validator)
|
91
65
|
@database.pool.connection_validation_timeout = @jdbc_validation_timeout
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-integration-jdbc'
|
3
|
-
s.version = '5.0.
|
3
|
+
s.version = '5.0.5'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Integration with JDBC - input and filter plugins"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/inputs/jdbc_spec.rb
CHANGED
@@ -13,9 +13,13 @@ require "date"
|
|
13
13
|
# We do not need to set TZ env var anymore because we can have 'Sequel.application_timezone' set to utc by default now.
|
14
14
|
|
15
15
|
describe LogStash::Inputs::Jdbc do
|
16
|
+
let(:connection_string) { "jdbc:derby:memory:testdb;create=true" }
|
16
17
|
let(:mixin_settings) do
|
17
|
-
{
|
18
|
-
|
18
|
+
{
|
19
|
+
"jdbc_user" => ENV['USER'],
|
20
|
+
"jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
|
21
|
+
"jdbc_connection_string" => connection_string
|
22
|
+
}
|
19
23
|
end
|
20
24
|
let(:settings) { {} }
|
21
25
|
let(:plugin) { LogStash::Inputs::Jdbc.new(mixin_settings.merge(settings)) }
|
@@ -112,6 +116,57 @@ describe LogStash::Inputs::Jdbc do
|
|
112
116
|
end
|
113
117
|
end
|
114
118
|
|
119
|
+
context "when sequel opts has user option" do
|
120
|
+
let(:settings) do
|
121
|
+
{
|
122
|
+
"jdbc_user" => 'system',
|
123
|
+
"statement" => "SELECT * from test_table",
|
124
|
+
"sequel_opts" => { "user" => 'from-opts' }
|
125
|
+
}
|
126
|
+
end
|
127
|
+
|
128
|
+
before do
|
129
|
+
plugin.register
|
130
|
+
end
|
131
|
+
|
132
|
+
after do
|
133
|
+
plugin.stop
|
134
|
+
end
|
135
|
+
|
136
|
+
it "should honor set jdbc-user when connecting" do
|
137
|
+
expect( Sequel ).to receive(:connect).with connection_string, hash_including(:user=>"system")
|
138
|
+
plugin.send(:jdbc_connect)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
context "with sequel opts" do
|
143
|
+
let(:settings) do
|
144
|
+
{
|
145
|
+
"jdbc_user" => 'system',
|
146
|
+
"statement" => "SELECT * from test_table",
|
147
|
+
"sequel_opts" => {
|
148
|
+
"truthy" => 'true',
|
149
|
+
"falsey" => 'false',
|
150
|
+
"foo" => 'bar',
|
151
|
+
"jdbc_properties" => { "some" => 'true' }
|
152
|
+
}
|
153
|
+
}
|
154
|
+
end
|
155
|
+
|
156
|
+
before do
|
157
|
+
plugin.register
|
158
|
+
end
|
159
|
+
|
160
|
+
after do
|
161
|
+
plugin.stop
|
162
|
+
end
|
163
|
+
|
164
|
+
it "should symbolize keys" do
|
165
|
+
expect( Sequel ).to receive(:connect).with connection_string,
|
166
|
+
hash_including(:truthy => true, :falsey => false, :foo => 'bar', :jdbc_properties => { 'some' => 'true' })
|
167
|
+
plugin.send(:jdbc_connect)
|
168
|
+
end
|
169
|
+
end
|
115
170
|
|
116
171
|
context "when neither statement and statement_filepath arguments are passed" do
|
117
172
|
it "should fail to register" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-integration-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.0.
|
4
|
+
version: 5.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-06-
|
11
|
+
date: 2020-06-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -234,6 +234,7 @@ files:
|
|
234
234
|
- lib/logstash/inputs/jdbc.rb
|
235
235
|
- lib/logstash/inputs/tzinfo_jruby_patch.rb
|
236
236
|
- lib/logstash/plugin_mixins/jdbc/checked_count_logger.rb
|
237
|
+
- lib/logstash/plugin_mixins/jdbc/common.rb
|
237
238
|
- lib/logstash/plugin_mixins/jdbc/jdbc.rb
|
238
239
|
- lib/logstash/plugin_mixins/jdbc/statement_handler.rb
|
239
240
|
- lib/logstash/plugin_mixins/jdbc/value_tracking.rb
|