logstash-output-jdbc 0.1.4 → 0.2.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +0 -1
- data/lib/logstash-output-jdbc_jars.rb +5 -0
- data/lib/logstash/outputs/jdbc.rb +65 -30
- data/logstash-output-jdbc.gemspec +3 -1
- data/vendor/jar-dependencies/runtime-jars/HikariCP-2.4.2.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar +0 -0
- data/vendor/jar-dependencies/runtime-jars/slf4j-nop-1.7.13.jar +0 -0
- metadata +33 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f4fb996001ccfc6faabb63f7d242ab88e176a8c1
|
4
|
+
data.tar.gz: e843a30f504af936835909f388dc7fe74c5e3d19
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6da31e801e832f0e9150a468fc07e8126a457069b9536f1add378baf8121f146a17fb08e83be87a6c2cb667b27ed2812262eace113059030537d58655ce06c7c
|
7
|
+
data.tar.gz: d5a84a95b71c98a512c6bf66135c3f8b3e68260da9d777a0be2c0628e649f9a7b8fc1cfdb8febdd22a6180090cf807c153b74b45f3fb6445d5f9fb699fc0e447
|
data/.gitignore
CHANGED
@@ -3,6 +3,7 @@ require "logstash/outputs/base"
|
|
3
3
|
require "logstash/namespace"
|
4
4
|
require "stud/buffer"
|
5
5
|
require "java"
|
6
|
+
require "logstash-output-jdbc_jars"
|
6
7
|
|
7
8
|
class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
8
9
|
# Adds buffer support
|
@@ -10,15 +11,31 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|
10
11
|
|
11
12
|
config_name "jdbc"
|
12
13
|
|
13
|
-
# Driver class
|
14
|
-
config :driver_class, :
|
14
|
+
# Driver class - No longer required
|
15
|
+
config :driver_class, :obsolete => true
|
15
16
|
|
16
|
-
#
|
17
|
+
# Where to find the jar
|
18
|
+
# Defaults to not required, and to the original behaviour
|
19
|
+
config :driver_jar_path, :validate => :string, :required => false
|
20
|
+
|
21
|
+
# jdbc connection string
|
17
22
|
config :connection_string, :validate => :string, :required => true
|
18
23
|
|
24
|
+
# jdbc username - optional, maybe in the connection string
|
25
|
+
config :username, :validate => :string, :required => false
|
26
|
+
|
27
|
+
# jdbc password - optional, maybe in the connection string
|
28
|
+
config :password, :validate => :string, :required => false
|
29
|
+
|
19
30
|
# [ "insert into table (message) values(?)", "%{message}" ]
|
20
31
|
config :statement, :validate => :array, :required => true
|
21
32
|
|
33
|
+
# Number of connections in the pool to maintain
|
34
|
+
config :max_pool_size, :validate => :number, :default => 5
|
35
|
+
|
36
|
+
# Connection timeout
|
37
|
+
config :connection_timeout, :validate => :number, :default => 2800
|
38
|
+
|
22
39
|
# We buffer a certain number of events before flushing that out to SQL.
|
23
40
|
# This setting controls how many events will be buffered before sending a
|
24
41
|
# batch of events.
|
@@ -40,7 +57,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|
40
57
|
|
41
58
|
# Maximum number of repeating (sequential) exceptions, before we stop retrying
|
42
59
|
# If set to < 1, then it will infinitely retry.
|
43
|
-
config :max_repeat_exceptions, :validate => :number, :default =>
|
60
|
+
config :max_repeat_exceptions, :validate => :number, :default => 4
|
44
61
|
|
45
62
|
# The max number of seconds since the last exception, before we consider it
|
46
63
|
# a different cause.
|
@@ -49,34 +66,21 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|
49
66
|
|
50
67
|
public
|
51
68
|
def register
|
52
|
-
|
53
69
|
@logger.info("JDBC - Starting up")
|
54
70
|
|
55
|
-
|
56
|
-
jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar")
|
57
|
-
else
|
58
|
-
jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
|
59
|
-
end
|
60
|
-
|
61
|
-
@logger.debug("JDBC - jarpath", path: jarpath)
|
62
|
-
|
63
|
-
jars = Dir[jarpath]
|
64
|
-
raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty?
|
65
|
-
|
66
|
-
jars.each do |jar|
|
67
|
-
@logger.debug("JDBC - Loaded jar", :jar => jar)
|
68
|
-
require jar
|
69
|
-
end
|
70
|
-
|
71
|
-
import @driver_class
|
71
|
+
load_jar_files!
|
72
72
|
|
73
|
-
|
74
|
-
@
|
73
|
+
@pool = Java::ComZaxxerHikari::HikariDataSource.new
|
74
|
+
@pool.setJdbcUrl(@connection_string)
|
75
|
+
|
76
|
+
@pool.setUsername(@username) if @username
|
77
|
+
@pool.setPassword(@password) if @password
|
75
78
|
|
76
|
-
@
|
79
|
+
@pool.setMaximumPoolSize(@max_pool_size)
|
80
|
+
@pool.setConnectionTimeout(@connection_timeout)
|
77
81
|
|
78
82
|
if (@flush_size > 1000)
|
79
|
-
@logger.warn("JDBC - Flush size is set to > 1000
|
83
|
+
@logger.warn("JDBC - Flush size is set to > 1000")
|
80
84
|
end
|
81
85
|
|
82
86
|
@repeat_exception_count = 0
|
@@ -101,7 +105,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|
101
105
|
end
|
102
106
|
|
103
107
|
def flush(events, teardown=false)
|
104
|
-
|
108
|
+
connection = @pool.getConnection()
|
109
|
+
|
110
|
+
statement = connection.prepareStatement(@statement[0])
|
105
111
|
|
106
112
|
events.each do |event|
|
107
113
|
next if @statement.length < 2
|
@@ -132,6 +138,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|
132
138
|
begin
|
133
139
|
@logger.debug("JDBC - Sending SQL", :sql => statement.toString())
|
134
140
|
statement.executeBatch()
|
141
|
+
statement.close()
|
135
142
|
rescue => e
|
136
143
|
# Raising an exception will incur a retry from Stud::Buffer.
|
137
144
|
# Since the exceutebatch failed this should mean any events failed to be
|
@@ -140,9 +147,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|
140
147
|
if e.getNextException() != nil
|
141
148
|
@logger.warn("JDBC - Exception. Will automatically retry", :exception => e.getNextException())
|
142
149
|
end
|
150
|
+
ensure
|
151
|
+
connection.close();
|
143
152
|
end
|
144
|
-
|
145
|
-
statement.close()
|
146
153
|
end
|
147
154
|
|
148
155
|
def on_flush_error(e)
|
@@ -165,8 +172,36 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
|
|
165
172
|
|
166
173
|
def teardown
|
167
174
|
buffer_flush(:final => true)
|
168
|
-
@
|
175
|
+
@pool.close()
|
169
176
|
super
|
170
177
|
end
|
171
178
|
|
179
|
+
private
|
180
|
+
|
181
|
+
def load_jar_files!
|
182
|
+
# Load jar from driver path
|
183
|
+
unless @driver_jar_path.nil?
|
184
|
+
raise Exception.new("JDBC - Could not find jar file at given path. Check config.") unless File.exists? @driver_jar_path
|
185
|
+
require @driver_jar_path
|
186
|
+
return
|
187
|
+
end
|
188
|
+
|
189
|
+
# Revert original behaviour of loading from vendor directory
|
190
|
+
# if no path given
|
191
|
+
if ENV['LOGSTASH_HOME']
|
192
|
+
jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar")
|
193
|
+
else
|
194
|
+
jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
|
195
|
+
end
|
196
|
+
|
197
|
+
@logger.debug("JDBC - jarpath", path: jarpath)
|
198
|
+
|
199
|
+
jars = Dir[jarpath]
|
200
|
+
raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty?
|
201
|
+
|
202
|
+
jars.each do |jar|
|
203
|
+
@logger.debug("JDBC - Loaded jar", :jar => jar)
|
204
|
+
require jar
|
205
|
+
end
|
206
|
+
end
|
172
207
|
end # class LogStash::Outputs::jdbc
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-jdbc'
|
3
|
-
s.version = "0.
|
3
|
+
s.version = "0.2.0.rc1"
|
4
4
|
s.licenses = [ "Apache License (2.0)" ]
|
5
5
|
s.summary = "This plugin allows you to output to SQL, via JDBC"
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
|
|
19
19
|
|
20
20
|
# Gem dependencies
|
21
21
|
s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
|
22
|
+
s.add_runtime_dependency 'stud'
|
22
23
|
s.add_runtime_dependency "logstash-codec-plain"
|
24
|
+
|
23
25
|
s.add_development_dependency "logstash-devutils"
|
24
26
|
end
|
Binary file
|
Binary file
|
Binary file
|
metadata
CHANGED
@@ -1,61 +1,75 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- the_angry_angel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-11-
|
11
|
+
date: 2015-11-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: 2.0.0.beta2
|
20
|
-
- -
|
20
|
+
- - <
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: 3.0.0
|
23
23
|
type: :runtime
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
|
-
- -
|
27
|
+
- - '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: 2.0.0.beta2
|
30
|
-
- -
|
30
|
+
- - <
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 3.0.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: stud
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - '>='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
name: logstash-codec-plain
|
35
49
|
requirement: !ruby/object:Gem::Requirement
|
36
50
|
requirements:
|
37
|
-
- -
|
51
|
+
- - '>='
|
38
52
|
- !ruby/object:Gem::Version
|
39
53
|
version: '0'
|
40
54
|
type: :runtime
|
41
55
|
prerelease: false
|
42
56
|
version_requirements: !ruby/object:Gem::Requirement
|
43
57
|
requirements:
|
44
|
-
- -
|
58
|
+
- - '>='
|
45
59
|
- !ruby/object:Gem::Version
|
46
60
|
version: '0'
|
47
61
|
- !ruby/object:Gem::Dependency
|
48
62
|
name: logstash-devutils
|
49
63
|
requirement: !ruby/object:Gem::Requirement
|
50
64
|
requirements:
|
51
|
-
- -
|
65
|
+
- - '>='
|
52
66
|
- !ruby/object:Gem::Version
|
53
67
|
version: '0'
|
54
68
|
type: :development
|
55
69
|
prerelease: false
|
56
70
|
version_requirements: !ruby/object:Gem::Requirement
|
57
71
|
requirements:
|
58
|
-
- -
|
72
|
+
- - '>='
|
59
73
|
- !ruby/object:Gem::Version
|
60
74
|
version: '0'
|
61
75
|
description: This gem is a logstash plugin required to be installed on top of the
|
@@ -66,13 +80,17 @@ executables: []
|
|
66
80
|
extensions: []
|
67
81
|
extra_rdoc_files: []
|
68
82
|
files:
|
69
|
-
-
|
83
|
+
- .gitignore
|
70
84
|
- Gemfile
|
71
85
|
- LICENSE.txt
|
72
86
|
- README.md
|
73
87
|
- Rakefile
|
88
|
+
- lib/logstash-output-jdbc_jars.rb
|
74
89
|
- lib/logstash/outputs/jdbc.rb
|
75
90
|
- logstash-output-jdbc.gemspec
|
91
|
+
- vendor/jar-dependencies/runtime-jars/HikariCP-2.4.2.jar
|
92
|
+
- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
|
93
|
+
- vendor/jar-dependencies/runtime-jars/slf4j-nop-1.7.13.jar
|
76
94
|
homepage: https://github.com/theangryangel/logstash-output-jdbc
|
77
95
|
licenses:
|
78
96
|
- Apache License (2.0)
|
@@ -85,17 +103,17 @@ require_paths:
|
|
85
103
|
- lib
|
86
104
|
required_ruby_version: !ruby/object:Gem::Requirement
|
87
105
|
requirements:
|
88
|
-
- -
|
106
|
+
- - '>='
|
89
107
|
- !ruby/object:Gem::Version
|
90
108
|
version: '0'
|
91
109
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
92
110
|
requirements:
|
93
|
-
- -
|
111
|
+
- - '>'
|
94
112
|
- !ruby/object:Gem::Version
|
95
|
-
version:
|
113
|
+
version: 1.3.1
|
96
114
|
requirements: []
|
97
115
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.
|
116
|
+
rubygems_version: 2.0.14
|
99
117
|
signing_key:
|
100
118
|
specification_version: 4
|
101
119
|
summary: This plugin allows you to output to SQL, via JDBC
|