logstash-input-jdbc 1.0.2 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -9
- data/lib/logstash/inputs/jdbc.rb +4 -3
- data/lib/logstash/plugin_mixins/jdbc.rb +1 -32
- data/logstash-input-jdbc.gemspec +3 -3
- data/spec/inputs/jdbc_spec.rb +24 -49
- metadata +29 -31
- data/.gitignore +0 -5
- data/Rakefile +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e56d7c26f413e58b553ccde453d22ec8245fdccd
|
4
|
+
data.tar.gz: 5d9253e023e204293b44758ee3f795fef2aa7726
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 129860f0076c9732fbd63d0ad9b3ec549d9ba7ed4018c8b1a59255350c7f2bd8825f2bc39fb3bc4932081ddec25449f35b5115a7c8210a3ec16fcb32fd2d400f
|
7
|
+
data.tar.gz: a9bade4dd5b1346b7b2282e83aa45cf756899d8e0ab51b1c036c346362a069e351878e959b723112a6ff791c92c845e4641069f699179eff84f27c1d8f4ca1d2
|
data/CHANGELOG.md
CHANGED
@@ -1,10 +1,7 @@
|
|
1
|
-
##
|
2
|
-
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
- Added catch-all configuration option for any other options that Sequel lib supports
|
8
|
-
|
9
|
-
## 1.0.0
|
1
|
+
## 2.0.0
|
2
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
|
+
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
|
+
- Dependency on logstash-core update to 2.0
|
5
|
+
|
6
|
+
* 1.0.0
|
10
7
|
- Initial release
|
data/lib/logstash/inputs/jdbc.rb
CHANGED
@@ -159,17 +159,18 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
|
|
159
159
|
|
160
160
|
def run(queue)
|
161
161
|
if @schedule
|
162
|
-
@scheduler = Rufus::Scheduler.new
|
162
|
+
@scheduler = Rufus::Scheduler.new
|
163
163
|
@scheduler.cron @schedule do
|
164
164
|
execute_query(queue)
|
165
165
|
end
|
166
|
+
|
166
167
|
@scheduler.join
|
167
168
|
else
|
168
169
|
execute_query(queue)
|
169
170
|
end
|
170
171
|
end # def run
|
171
172
|
|
172
|
-
def
|
173
|
+
def stop
|
173
174
|
@scheduler.stop if @scheduler
|
174
175
|
|
175
176
|
# update state file for next run
|
@@ -178,7 +179,7 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
|
|
178
179
|
end
|
179
180
|
|
180
181
|
close_jdbc_connection
|
181
|
-
end
|
182
|
+
end
|
182
183
|
|
183
184
|
private
|
184
185
|
|
@@ -57,37 +57,6 @@ module LogStash::PluginMixins::Jdbc
|
|
57
57
|
# Connection pool configuration.
|
58
58
|
# How often to validate a connection (in seconds)
|
59
59
|
config :jdbc_validation_timeout, :validate => :number, :default => 3600
|
60
|
-
|
61
|
-
# Connection pool configuration.
|
62
|
-
# The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5)
|
63
|
-
config :jdbc_pool_timeout, :validate => :number, :default => 5
|
64
|
-
|
65
|
-
# General/Vendor-specific Sequel configuration options.
|
66
|
-
#
|
67
|
-
# An example of an optional connection pool configuration
|
68
|
-
# max_connections - The maximum number of connections the connection pool
|
69
|
-
#
|
70
|
-
# examples of vendor-specific options can be found in this
|
71
|
-
# documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc
|
72
|
-
config :sequel_opts, :validate => :hash, :default => {}
|
73
|
-
end
|
74
|
-
|
75
|
-
private
|
76
|
-
def jdbc_connect
|
77
|
-
opts = {
|
78
|
-
:user => @jdbc_user,
|
79
|
-
:password => @jdbc_password.nil? ? nil : @jdbc_password.value,
|
80
|
-
:pool_timeout => @jdbc_pool_timeout
|
81
|
-
}.merge(@sequel_opts)
|
82
|
-
begin
|
83
|
-
Sequel.connect(@jdbc_connection_string, opts=opts)
|
84
|
-
rescue Sequel::PoolTimeout => e
|
85
|
-
@logger.error("Failed to connect to database. #{@jdbc_pool_timeout} second timeout exceeded.")
|
86
|
-
raise e
|
87
|
-
rescue Sequel::Error => e
|
88
|
-
@logger.error("Unable to connect to database", :error_message => e.message)
|
89
|
-
raise e
|
90
|
-
end
|
91
60
|
end
|
92
61
|
|
93
62
|
public
|
@@ -107,7 +76,7 @@ module LogStash::PluginMixins::Jdbc
|
|
107
76
|
end
|
108
77
|
raise LogStash::ConfigurationError, "#{e}. #{message}"
|
109
78
|
end
|
110
|
-
@database =
|
79
|
+
@database = Sequel.connect(@jdbc_connection_string, :user=> @jdbc_user, :password=> @jdbc_password.nil? ? nil : @jdbc_password.value)
|
111
80
|
@database.extension(:pagination)
|
112
81
|
if @jdbc_validate_connection
|
113
82
|
@database.extension(:connection_validator)
|
data/logstash-input-jdbc.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-input-jdbc'
|
3
|
-
s.version = '
|
3
|
+
s.version = '2.0.1'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "This example input streams a string at a definable interval."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
# Files
|
13
|
-
s.files =
|
13
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
14
14
|
# Tests
|
15
15
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
16
|
|
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
|
19
19
|
|
20
20
|
# Gem dependencies
|
21
|
-
s.add_runtime_dependency
|
21
|
+
s.add_runtime_dependency "logstash-core", ">= 2.0.0.snapshot", "< 3.0.0"
|
22
22
|
s.add_runtime_dependency 'logstash-codec-plain'
|
23
23
|
s.add_runtime_dependency 'sequel'
|
24
24
|
s.add_runtime_dependency 'rufus-scheduler'
|
data/spec/inputs/jdbc_spec.rb
CHANGED
@@ -6,8 +6,7 @@ require "sequel/adapters/jdbc"
|
|
6
6
|
require "timecop"
|
7
7
|
require "stud/temporary"
|
8
8
|
|
9
|
-
|
10
|
-
describe "jdbc" do
|
9
|
+
describe LogStash::Inputs::Jdbc do
|
11
10
|
let(:mixin_settings) { {"jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver", "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"} }
|
12
11
|
let(:settings) { {} }
|
13
12
|
let(:plugin) { LogStash::Inputs::Jdbc.new(mixin_settings.merge(settings)) }
|
@@ -33,13 +32,22 @@ describe "jdbc" do
|
|
33
32
|
|
34
33
|
it "should register without raising exception" do
|
35
34
|
expect { plugin.register }.to_not raise_error
|
36
|
-
plugin.
|
35
|
+
plugin.stop
|
37
36
|
end
|
38
37
|
|
39
|
-
it "should
|
40
|
-
|
41
|
-
expect { plugin.
|
42
|
-
|
38
|
+
it "should stop without raising exception" do
|
39
|
+
plugin.register
|
40
|
+
expect { plugin.stop }.to_not raise_error
|
41
|
+
end
|
42
|
+
|
43
|
+
it_behaves_like "an interruptible input plugin" do
|
44
|
+
let(:settings) do
|
45
|
+
{
|
46
|
+
"statement" => "SELECT 1 FROM test_table",
|
47
|
+
"schedule" => "* * * * * UTC"
|
48
|
+
}
|
49
|
+
end
|
50
|
+
let(:config) { mixin_settings.merge(settings) }
|
43
51
|
end
|
44
52
|
end
|
45
53
|
|
@@ -70,7 +78,7 @@ describe "jdbc" do
|
|
70
78
|
end
|
71
79
|
|
72
80
|
after do
|
73
|
-
plugin.
|
81
|
+
plugin.stop
|
74
82
|
end
|
75
83
|
|
76
84
|
it "should read in statement from file" do
|
@@ -91,7 +99,7 @@ describe "jdbc" do
|
|
91
99
|
end
|
92
100
|
|
93
101
|
after do
|
94
|
-
plugin.
|
102
|
+
plugin.stop
|
95
103
|
end
|
96
104
|
|
97
105
|
it "should retrieve params correctly from Event" do
|
@@ -114,7 +122,7 @@ describe "jdbc" do
|
|
114
122
|
plugin.run(queue)
|
115
123
|
end
|
116
124
|
sleep 3
|
117
|
-
plugin.
|
125
|
+
plugin.stop
|
118
126
|
runner.kill
|
119
127
|
runner.join
|
120
128
|
expect(queue.size).to eq(2)
|
@@ -140,7 +148,7 @@ describe "jdbc" do
|
|
140
148
|
end
|
141
149
|
|
142
150
|
after do
|
143
|
-
plugin.
|
151
|
+
plugin.stop
|
144
152
|
end
|
145
153
|
|
146
154
|
it "should fetch all rows" do
|
@@ -167,7 +175,7 @@ describe "jdbc" do
|
|
167
175
|
end
|
168
176
|
|
169
177
|
after do
|
170
|
-
plugin.
|
178
|
+
plugin.stop
|
171
179
|
end
|
172
180
|
|
173
181
|
it "should successfully iterate table with respect to field values" do
|
@@ -206,7 +214,7 @@ describe "jdbc" do
|
|
206
214
|
end
|
207
215
|
|
208
216
|
after do
|
209
|
-
plugin.
|
217
|
+
plugin.stop
|
210
218
|
end
|
211
219
|
|
212
220
|
it "should respect last run metadata" do
|
@@ -232,7 +240,7 @@ describe "jdbc" do
|
|
232
240
|
end
|
233
241
|
|
234
242
|
after do
|
235
|
-
plugin.
|
243
|
+
plugin.stop
|
236
244
|
end
|
237
245
|
|
238
246
|
it "should ignore last run metadata if :clean_run set to true" do
|
@@ -254,7 +262,7 @@ describe "jdbc" do
|
|
254
262
|
end
|
255
263
|
|
256
264
|
after do
|
257
|
-
plugin.
|
265
|
+
plugin.stop
|
258
266
|
end
|
259
267
|
|
260
268
|
it "should not save state if :record_last_run is false" do
|
@@ -282,7 +290,7 @@ describe "jdbc" do
|
|
282
290
|
end
|
283
291
|
|
284
292
|
after do
|
285
|
-
plugin.
|
293
|
+
plugin.stop
|
286
294
|
end
|
287
295
|
|
288
296
|
it "should fetch all rows" do
|
@@ -302,37 +310,4 @@ describe "jdbc" do
|
|
302
310
|
expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
|
303
311
|
end
|
304
312
|
end
|
305
|
-
|
306
|
-
context "when timing out on connection" do
|
307
|
-
let(:settings) do
|
308
|
-
{
|
309
|
-
"statement" => "SELECT * FROM test_table",
|
310
|
-
"jdbc_pool_timeout" => 0,
|
311
|
-
"jdbc_connection_string" => 'mock://localhost:1527/db',
|
312
|
-
"sequel_opts" => {
|
313
|
-
"max_connections" => 1
|
314
|
-
}
|
315
|
-
}
|
316
|
-
end
|
317
|
-
|
318
|
-
it "should raise PoolTimeout error" do
|
319
|
-
plugin.register
|
320
|
-
db = plugin.instance_variable_get(:@database)
|
321
|
-
expect(db.pool.instance_variable_get(:@timeout)).to eq(0)
|
322
|
-
expect(db.pool.instance_variable_get(:@max_size)).to eq(1)
|
323
|
-
|
324
|
-
q, q1 = Queue.new, Queue.new
|
325
|
-
t = Thread.new{db.pool.hold{|c| q1.push nil; q.pop}}
|
326
|
-
q1.pop
|
327
|
-
expect{db.pool.hold {|c|}}.to raise_error(Sequel::PoolTimeout)
|
328
|
-
q.push nil
|
329
|
-
t.join
|
330
|
-
end
|
331
|
-
|
332
|
-
it "should log error message" do
|
333
|
-
allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
|
334
|
-
expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded.")
|
335
|
-
expect { plugin.register }.to raise_error(Sequel::PoolTimeout)
|
336
|
-
end
|
337
|
-
end
|
338
313
|
end
|
metadata
CHANGED
@@ -1,146 +1,144 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-10-
|
11
|
+
date: 2015-10-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: logstash-core
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - '>='
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 1.4.0
|
20
|
-
- - <
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: 2.0.0
|
23
14
|
requirement: !ruby/object:Gem::Requirement
|
24
15
|
requirements:
|
25
16
|
- - '>='
|
26
17
|
- !ruby/object:Gem::Version
|
27
|
-
version:
|
18
|
+
version: 2.0.0.snapshot
|
28
19
|
- - <
|
29
20
|
- !ruby/object:Gem::Version
|
30
|
-
version:
|
21
|
+
version: 3.0.0
|
22
|
+
name: logstash-core
|
31
23
|
prerelease: false
|
32
24
|
type: :runtime
|
33
|
-
- !ruby/object:Gem::Dependency
|
34
|
-
name: logstash-codec-plain
|
35
25
|
version_requirements: !ruby/object:Gem::Requirement
|
36
26
|
requirements:
|
37
27
|
- - '>='
|
38
28
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
29
|
+
version: 2.0.0.snapshot
|
30
|
+
- - <
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 3.0.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
40
34
|
requirement: !ruby/object:Gem::Requirement
|
41
35
|
requirements:
|
42
36
|
- - '>='
|
43
37
|
- !ruby/object:Gem::Version
|
44
38
|
version: '0'
|
39
|
+
name: logstash-codec-plain
|
45
40
|
prerelease: false
|
46
41
|
type: :runtime
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
|
-
name: sequel
|
49
42
|
version_requirements: !ruby/object:Gem::Requirement
|
50
43
|
requirements:
|
51
44
|
- - '>='
|
52
45
|
- !ruby/object:Gem::Version
|
53
46
|
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
54
48
|
requirement: !ruby/object:Gem::Requirement
|
55
49
|
requirements:
|
56
50
|
- - '>='
|
57
51
|
- !ruby/object:Gem::Version
|
58
52
|
version: '0'
|
53
|
+
name: sequel
|
59
54
|
prerelease: false
|
60
55
|
type: :runtime
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
|
-
name: rufus-scheduler
|
63
56
|
version_requirements: !ruby/object:Gem::Requirement
|
64
57
|
requirements:
|
65
58
|
- - '>='
|
66
59
|
- !ruby/object:Gem::Version
|
67
60
|
version: '0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
68
62
|
requirement: !ruby/object:Gem::Requirement
|
69
63
|
requirements:
|
70
64
|
- - '>='
|
71
65
|
- !ruby/object:Gem::Version
|
72
66
|
version: '0'
|
67
|
+
name: rufus-scheduler
|
73
68
|
prerelease: false
|
74
69
|
type: :runtime
|
75
|
-
- !ruby/object:Gem::Dependency
|
76
|
-
name: logstash-devutils
|
77
70
|
version_requirements: !ruby/object:Gem::Requirement
|
78
71
|
requirements:
|
79
72
|
- - '>='
|
80
73
|
- !ruby/object:Gem::Version
|
81
74
|
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
82
76
|
requirement: !ruby/object:Gem::Requirement
|
83
77
|
requirements:
|
84
78
|
- - '>='
|
85
79
|
- !ruby/object:Gem::Version
|
86
80
|
version: '0'
|
81
|
+
name: logstash-devutils
|
87
82
|
prerelease: false
|
88
83
|
type: :development
|
89
|
-
- !ruby/object:Gem::Dependency
|
90
|
-
name: timecop
|
91
84
|
version_requirements: !ruby/object:Gem::Requirement
|
92
85
|
requirements:
|
93
86
|
- - '>='
|
94
87
|
- !ruby/object:Gem::Version
|
95
88
|
version: '0'
|
89
|
+
- !ruby/object:Gem::Dependency
|
96
90
|
requirement: !ruby/object:Gem::Requirement
|
97
91
|
requirements:
|
98
92
|
- - '>='
|
99
93
|
- !ruby/object:Gem::Version
|
100
94
|
version: '0'
|
95
|
+
name: timecop
|
101
96
|
prerelease: false
|
102
97
|
type: :development
|
103
|
-
- !ruby/object:Gem::Dependency
|
104
|
-
name: jdbc-derby
|
105
98
|
version_requirements: !ruby/object:Gem::Requirement
|
106
99
|
requirements:
|
107
100
|
- - '>='
|
108
101
|
- !ruby/object:Gem::Version
|
109
102
|
version: '0'
|
103
|
+
- !ruby/object:Gem::Dependency
|
110
104
|
requirement: !ruby/object:Gem::Requirement
|
111
105
|
requirements:
|
112
106
|
- - '>='
|
113
107
|
- !ruby/object:Gem::Version
|
114
108
|
version: '0'
|
109
|
+
name: jdbc-derby
|
115
110
|
prerelease: false
|
116
111
|
type: :development
|
117
|
-
- !ruby/object:Gem::Dependency
|
118
|
-
name: docker-api
|
119
112
|
version_requirements: !ruby/object:Gem::Requirement
|
120
113
|
requirements:
|
121
114
|
- - '>='
|
122
115
|
- !ruby/object:Gem::Version
|
123
116
|
version: '0'
|
117
|
+
- !ruby/object:Gem::Dependency
|
124
118
|
requirement: !ruby/object:Gem::Requirement
|
125
119
|
requirements:
|
126
120
|
- - '>='
|
127
121
|
- !ruby/object:Gem::Version
|
128
122
|
version: '0'
|
123
|
+
name: docker-api
|
129
124
|
prerelease: false
|
130
125
|
type: :development
|
126
|
+
version_requirements: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - '>='
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
131
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
132
132
|
email: info@elastic.co
|
133
133
|
executables: []
|
134
134
|
extensions: []
|
135
135
|
extra_rdoc_files: []
|
136
136
|
files:
|
137
|
-
- .gitignore
|
138
137
|
- CHANGELOG.md
|
139
138
|
- Gemfile
|
140
139
|
- LICENSE
|
141
140
|
- NOTICE.TXT
|
142
141
|
- README.md
|
143
|
-
- Rakefile
|
144
142
|
- lib/logstash/inputs/jdbc.rb
|
145
143
|
- lib/logstash/plugin_mixins/jdbc.rb
|
146
144
|
- logstash-input-jdbc.gemspec
|
@@ -167,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
167
165
|
version: '0'
|
168
166
|
requirements: []
|
169
167
|
rubyforge_project:
|
170
|
-
rubygems_version: 2.4.
|
168
|
+
rubygems_version: 2.4.8
|
171
169
|
signing_key:
|
172
170
|
specification_version: 4
|
173
171
|
summary: This example input streams a string at a definable interval.
|
data/.gitignore
DELETED
data/Rakefile
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require "logstash/devutils/rake"
|