logstash-input-jdbc 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/inputs/jdbc.rb +13 -0
- data/lib/logstash/plugin_mixins/jdbc.rb +26 -8
- data/logstash-input-jdbc.gemspec +2 -2
- data/spec/inputs/jdbc_spec.rb +20 -1
- metadata +49 -55
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 210deedb95591e0f409fc3461af2d79bb8a384e8
|
4
|
+
data.tar.gz: 8543b8a26df1cd6a80fcace0f7c7e54be175cf2e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: df9c7fa5e10be322919f607a815c9fe899f05da6bb348c599e2d35390f5a10b1647ea0fe8ef60c8c05cfa524b7db3e39ae89fa256f832b5972f467c17a72c462
|
7
|
+
data.tar.gz: d42eccdbab0331fc8e7e834e45fb0944e22eb9e1eb583782955bf34aa7cad8aab33fe9c6c9b29f43257992f11e4e5947cd177f83d4e10cc9fb6071d80880725a
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
# 3.0.2
|
2
|
+
- Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
|
3
|
+
# 3.0.1
|
4
|
+
- New dependency requirements for logstash-core for the 5.0 release
|
1
5
|
## 3.0.0
|
2
6
|
- [#57](https://github.com/logstash-plugins/logstash-input-jdbc/issues/57) New feature: Allow tracking by a column value rather than by last run time. **This is a breaking change**, as users may be required to change from using `sql_last_start` to use `sql_last_value` in their queries. No other changes are required if you've been using time-based queries. See the documentation if you wish to use an incremental column value to track updates to your tables.
|
3
7
|
|
data/lib/logstash/inputs/jdbc.rb
CHANGED
@@ -95,6 +95,19 @@ require "yaml" # persistence
|
|
95
95
|
# `tracking_column` is set. It is updated accordingly after subsequent queries are run.
|
96
96
|
# |==========================================================
|
97
97
|
#
|
98
|
+
# Example:
|
99
|
+
# [source,ruby]
|
100
|
+
# ----------------------------------
|
101
|
+
# input {
|
102
|
+
# jdbc {
|
103
|
+
# statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value"
|
104
|
+
# use_column_value => true
|
105
|
+
# tracking_column => id
|
106
|
+
# # ... other configuration bits
|
107
|
+
# }
|
108
|
+
# }
|
109
|
+
# ----------------------------------
|
110
|
+
#
|
98
111
|
class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
|
99
112
|
include LogStash::PluginMixins::Jdbc
|
100
113
|
config_name "jdbc"
|
@@ -88,6 +88,11 @@ module LogStash::PluginMixins::Jdbc
|
|
88
88
|
# Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn,
|
89
89
|
# info and debug. The default value is info.
|
90
90
|
config :sql_log_level, :validate => [ "fatal", "error", "warn", "info", "debug" ], :default => "info"
|
91
|
+
|
92
|
+
# Maximum number of times to try connecting to database
|
93
|
+
config :connection_retry_attempts, :validate => :number, :default => 1
|
94
|
+
# Number of seconds to sleep between connection attempts
|
95
|
+
config :connection_retry_attempts_wait_time, :validate => :number, :default => 0.5
|
91
96
|
end
|
92
97
|
|
93
98
|
private
|
@@ -97,14 +102,27 @@ module LogStash::PluginMixins::Jdbc
|
|
97
102
|
:password => @jdbc_password.nil? ? nil : @jdbc_password.value,
|
98
103
|
:pool_timeout => @jdbc_pool_timeout
|
99
104
|
}.merge(@sequel_opts)
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
105
|
+
retry_attempts = @connection_retry_attempts
|
106
|
+
loop do
|
107
|
+
retry_attempts -= 1
|
108
|
+
begin
|
109
|
+
return Sequel.connect(@jdbc_connection_string, opts=opts)
|
110
|
+
rescue Sequel::PoolTimeout => e
|
111
|
+
if retry_attempts <= 0
|
112
|
+
@logger.error("Failed to connect to database. #{@jdbc_pool_timeout} second timeout exceeded. Tried #{@connection_retry_attempts} times.")
|
113
|
+
raise e
|
114
|
+
else
|
115
|
+
@logger.error("Failed to connect to database. #{@jdbc_pool_timeout} second timeout exceeded. Trying again.")
|
116
|
+
end
|
117
|
+
rescue Sequel::Error => e
|
118
|
+
if retry_attempts <= 0
|
119
|
+
@logger.error("Unable to connect to database. Tried #{@connection_retry_attempts} times", :error_message => e.message, )
|
120
|
+
raise e
|
121
|
+
else
|
122
|
+
@logger.error("Unable to connect to database. Trying again", :error_message => e.message)
|
123
|
+
end
|
124
|
+
end
|
125
|
+
sleep(@connection_retry_attempts_wait_time)
|
108
126
|
end
|
109
127
|
end
|
110
128
|
|
data/logstash-input-jdbc.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-jdbc'
|
3
|
-
s.version = '3.0.
|
3
|
+
s.version = '3.0.2'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "This example input streams a string at a definable interval."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
19
19
|
|
20
20
|
# Gem dependencies
|
21
|
-
s.add_runtime_dependency "logstash-core", "
|
21
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
|
22
22
|
s.add_runtime_dependency 'logstash-codec-plain'
|
23
23
|
s.add_runtime_dependency 'sequel'
|
24
24
|
s.add_runtime_dependency 'tzinfo'
|
data/spec/inputs/jdbc_spec.rb
CHANGED
@@ -746,7 +746,7 @@ describe LogStash::Inputs::Jdbc do
|
|
746
746
|
|
747
747
|
it "should log error message" do
|
748
748
|
allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
|
749
|
-
expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded.")
|
749
|
+
expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 1 times.")
|
750
750
|
expect { plugin.register }.to raise_error(Sequel::PoolTimeout)
|
751
751
|
end
|
752
752
|
end
|
@@ -818,4 +818,23 @@ describe LogStash::Inputs::Jdbc do
|
|
818
818
|
end
|
819
819
|
end
|
820
820
|
end
|
821
|
+
|
822
|
+
context "when specifying connection_retry_attempts" do
|
823
|
+
let(:settings) { {"statement" => "SELECT 1 as col1 FROM test_table"} }
|
824
|
+
|
825
|
+
it "should try to connect connection_retry_attempts times" do
|
826
|
+
mixin_settings['connection_retry_attempts'] = 2
|
827
|
+
mixin_settings['jdbc_pool_timeout'] = 0
|
828
|
+
allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
|
829
|
+
expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Trying again.")
|
830
|
+
expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 2 times.")
|
831
|
+
expect { plugin.register }.to raise_error(Sequel::PoolTimeout)
|
832
|
+
end
|
833
|
+
|
834
|
+
it "should not fail when passed a non-positive value" do
|
835
|
+
mixin_settings['connection_retry_attempts'] = -2
|
836
|
+
expect { plugin.register }.to_not raise_error
|
837
|
+
plugin.stop
|
838
|
+
end
|
839
|
+
end
|
821
840
|
end
|
metadata
CHANGED
@@ -1,161 +1,155 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: logstash-core
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - '>='
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 2.0.0.beta2
|
20
|
-
- - <
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 3.0.0
|
23
14
|
requirement: !ruby/object:Gem::Requirement
|
24
15
|
requirements:
|
25
|
-
- -
|
26
|
-
- !ruby/object:Gem::Version
|
27
|
-
version: 2.0.0.beta2
|
28
|
-
- - <
|
16
|
+
- - "~>"
|
29
17
|
- !ruby/object:Gem::Version
|
30
|
-
version:
|
18
|
+
version: '1.0'
|
19
|
+
name: logstash-core-plugin-api
|
31
20
|
prerelease: false
|
32
21
|
type: :runtime
|
33
|
-
- !ruby/object:Gem::Dependency
|
34
|
-
name: logstash-codec-plain
|
35
22
|
version_requirements: !ruby/object:Gem::Requirement
|
36
23
|
requirements:
|
37
|
-
- -
|
24
|
+
- - "~>"
|
38
25
|
- !ruby/object:Gem::Version
|
39
|
-
version: '0'
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
40
28
|
requirement: !ruby/object:Gem::Requirement
|
41
29
|
requirements:
|
42
|
-
- -
|
30
|
+
- - ">="
|
43
31
|
- !ruby/object:Gem::Version
|
44
32
|
version: '0'
|
33
|
+
name: logstash-codec-plain
|
45
34
|
prerelease: false
|
46
35
|
type: :runtime
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
|
-
name: sequel
|
49
36
|
version_requirements: !ruby/object:Gem::Requirement
|
50
37
|
requirements:
|
51
|
-
- -
|
38
|
+
- - ">="
|
52
39
|
- !ruby/object:Gem::Version
|
53
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
54
42
|
requirement: !ruby/object:Gem::Requirement
|
55
43
|
requirements:
|
56
|
-
- -
|
44
|
+
- - ">="
|
57
45
|
- !ruby/object:Gem::Version
|
58
46
|
version: '0'
|
47
|
+
name: sequel
|
59
48
|
prerelease: false
|
60
49
|
type: :runtime
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
|
-
name: tzinfo
|
63
50
|
version_requirements: !ruby/object:Gem::Requirement
|
64
51
|
requirements:
|
65
|
-
- -
|
52
|
+
- - ">="
|
66
53
|
- !ruby/object:Gem::Version
|
67
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
68
56
|
requirement: !ruby/object:Gem::Requirement
|
69
57
|
requirements:
|
70
|
-
- -
|
58
|
+
- - ">="
|
71
59
|
- !ruby/object:Gem::Version
|
72
60
|
version: '0'
|
61
|
+
name: tzinfo
|
73
62
|
prerelease: false
|
74
63
|
type: :runtime
|
75
|
-
- !ruby/object:Gem::Dependency
|
76
|
-
name: tzinfo-data
|
77
64
|
version_requirements: !ruby/object:Gem::Requirement
|
78
65
|
requirements:
|
79
|
-
- -
|
66
|
+
- - ">="
|
80
67
|
- !ruby/object:Gem::Version
|
81
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
82
70
|
requirement: !ruby/object:Gem::Requirement
|
83
71
|
requirements:
|
84
|
-
- -
|
72
|
+
- - ">="
|
85
73
|
- !ruby/object:Gem::Version
|
86
74
|
version: '0'
|
75
|
+
name: tzinfo-data
|
87
76
|
prerelease: false
|
88
77
|
type: :runtime
|
89
|
-
- !ruby/object:Gem::Dependency
|
90
|
-
name: rufus-scheduler
|
91
78
|
version_requirements: !ruby/object:Gem::Requirement
|
92
79
|
requirements:
|
93
|
-
- -
|
80
|
+
- - ">="
|
94
81
|
- !ruby/object:Gem::Version
|
95
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
96
84
|
requirement: !ruby/object:Gem::Requirement
|
97
85
|
requirements:
|
98
|
-
- -
|
86
|
+
- - ">="
|
99
87
|
- !ruby/object:Gem::Version
|
100
88
|
version: '0'
|
89
|
+
name: rufus-scheduler
|
101
90
|
prerelease: false
|
102
91
|
type: :runtime
|
103
|
-
- !ruby/object:Gem::Dependency
|
104
|
-
name: logstash-devutils
|
105
92
|
version_requirements: !ruby/object:Gem::Requirement
|
106
93
|
requirements:
|
107
|
-
- -
|
94
|
+
- - ">="
|
108
95
|
- !ruby/object:Gem::Version
|
109
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
110
98
|
requirement: !ruby/object:Gem::Requirement
|
111
99
|
requirements:
|
112
|
-
- -
|
100
|
+
- - ">="
|
113
101
|
- !ruby/object:Gem::Version
|
114
102
|
version: '0'
|
103
|
+
name: logstash-devutils
|
115
104
|
prerelease: false
|
116
105
|
type: :development
|
117
|
-
- !ruby/object:Gem::Dependency
|
118
|
-
name: timecop
|
119
106
|
version_requirements: !ruby/object:Gem::Requirement
|
120
107
|
requirements:
|
121
|
-
- -
|
108
|
+
- - ">="
|
122
109
|
- !ruby/object:Gem::Version
|
123
110
|
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
124
112
|
requirement: !ruby/object:Gem::Requirement
|
125
113
|
requirements:
|
126
|
-
- -
|
114
|
+
- - ">="
|
127
115
|
- !ruby/object:Gem::Version
|
128
116
|
version: '0'
|
117
|
+
name: timecop
|
129
118
|
prerelease: false
|
130
119
|
type: :development
|
131
|
-
- !ruby/object:Gem::Dependency
|
132
|
-
name: jdbc-derby
|
133
120
|
version_requirements: !ruby/object:Gem::Requirement
|
134
121
|
requirements:
|
135
|
-
- -
|
122
|
+
- - ">="
|
136
123
|
- !ruby/object:Gem::Version
|
137
124
|
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
138
126
|
requirement: !ruby/object:Gem::Requirement
|
139
127
|
requirements:
|
140
|
-
- -
|
128
|
+
- - ">="
|
141
129
|
- !ruby/object:Gem::Version
|
142
130
|
version: '0'
|
131
|
+
name: jdbc-derby
|
143
132
|
prerelease: false
|
144
133
|
type: :development
|
145
|
-
- !ruby/object:Gem::Dependency
|
146
|
-
name: docker-api
|
147
134
|
version_requirements: !ruby/object:Gem::Requirement
|
148
135
|
requirements:
|
149
|
-
- -
|
136
|
+
- - ">="
|
150
137
|
- !ruby/object:Gem::Version
|
151
138
|
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
152
140
|
requirement: !ruby/object:Gem::Requirement
|
153
141
|
requirements:
|
154
|
-
- -
|
142
|
+
- - ">="
|
155
143
|
- !ruby/object:Gem::Version
|
156
144
|
version: '0'
|
145
|
+
name: docker-api
|
157
146
|
prerelease: false
|
158
147
|
type: :development
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
159
153
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
160
154
|
email: info@elastic.co
|
161
155
|
executables: []
|
@@ -183,12 +177,12 @@ require_paths:
|
|
183
177
|
- lib
|
184
178
|
required_ruby_version: !ruby/object:Gem::Requirement
|
185
179
|
requirements:
|
186
|
-
- -
|
180
|
+
- - ">="
|
187
181
|
- !ruby/object:Gem::Version
|
188
182
|
version: '0'
|
189
183
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
190
184
|
requirements:
|
191
|
-
- -
|
185
|
+
- - ">="
|
192
186
|
- !ruby/object:Gem::Version
|
193
187
|
version: '0'
|
194
188
|
requirements: []
|