logstash-output-jdbc 0.1.4 → 0.2.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6ceaf64925ffd4d57cbf2fb0a44ef53c1898ae3a
4
- data.tar.gz: 21d2b8dfad9c8ab3f8dbfe74617d0fcf7cad7ffa
3
+ metadata.gz: f4fb996001ccfc6faabb63f7d242ab88e176a8c1
4
+ data.tar.gz: e843a30f504af936835909f388dc7fe74c5e3d19
5
5
  SHA512:
6
- metadata.gz: c6dda6cbf171dbe2cca8a6d63d9d8524a29fb420975a726e220620c38fc043c523d27ee33c37ce7ddf0521fa0eadff2adfa31e9ba2665238a28adc46a53ebb8c
7
- data.tar.gz: 168b6505cbb66157300a60c3009a17946afc72b45c17f4572476e962e43c1b14042622b49d98c59d7eefc63be85323f091d99f320f096a6d8c3d9563519c0209
6
+ metadata.gz: 6da31e801e832f0e9150a468fc07e8126a457069b9536f1add378baf8121f146a17fb08e83be87a6c2cb667b27ed2812262eace113059030537d58655ce06c7c
7
+ data.tar.gz: d5a84a95b71c98a512c6bf66135c3f8b3e68260da9d777a0be2c0628e649f9a7b8fc1cfdb8febdd22a6180090cf807c153b74b45f3fb6445d5f9fb699fc0e447
data/.gitignore CHANGED
@@ -2,4 +2,3 @@
2
2
  Gemfile.lock
3
3
  Gemfile.bak
4
4
  .bundle
5
- vendor
@@ -0,0 +1,5 @@
1
+ # encoding: utf-8
2
+ require 'logstash/environment'
3
+
4
+ root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
5
+ LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")
@@ -3,6 +3,7 @@ require "logstash/outputs/base"
3
3
  require "logstash/namespace"
4
4
  require "stud/buffer"
5
5
  require "java"
6
+ require "logstash-output-jdbc_jars"
6
7
 
7
8
  class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
8
9
  # Adds buffer support
@@ -10,15 +11,31 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
10
11
 
11
12
  config_name "jdbc"
12
13
 
13
- # Driver class
14
- config :driver_class, :validate => :string
14
+ # Driver class - No longer required
15
+ config :driver_class, :obsolete => true
15
16
 
16
- # connection string
17
+ # Where to find the jar
18
+ # Defaults to not required, and to the original behaviour
19
+ config :driver_jar_path, :validate => :string, :required => false
20
+
21
+ # jdbc connection string
17
22
  config :connection_string, :validate => :string, :required => true
18
23
 
24
+ # jdbc username - optional, maybe in the connection string
25
+ config :username, :validate => :string, :required => false
26
+
27
+ # jdbc password - optional, maybe in the connection string
28
+ config :password, :validate => :string, :required => false
29
+
19
30
  # [ "insert into table (message) values(?)", "%{message}" ]
20
31
  config :statement, :validate => :array, :required => true
21
32
 
33
+ # Number of connections in the pool to maintain
34
+ config :max_pool_size, :validate => :number, :default => 5
35
+
36
+ # Connection timeout
37
+ config :connection_timeout, :validate => :number, :default => 2800
38
+
22
39
  # We buffer a certain number of events before flushing that out to SQL.
23
40
  # This setting controls how many events will be buffered before sending a
24
41
  # batch of events.
@@ -40,7 +57,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
40
57
 
41
58
  # Maximum number of repeating (sequential) exceptions, before we stop retrying
42
59
  # If set to < 1, then it will infinitely retry.
43
- config :max_repeat_exceptions, :validate => :number, :default => 5
60
+ config :max_repeat_exceptions, :validate => :number, :default => 4
44
61
 
45
62
  # The max number of seconds since the last exception, before we consider it
46
63
  # a different cause.
@@ -49,34 +66,21 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
49
66
 
50
67
  public
51
68
  def register
52
-
53
69
  @logger.info("JDBC - Starting up")
54
70
 
55
- if ENV['LOGSTASH_HOME']
56
- jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar")
57
- else
58
- jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
59
- end
60
-
61
- @logger.debug("JDBC - jarpath", path: jarpath)
62
-
63
- jars = Dir[jarpath]
64
- raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty?
65
-
66
- jars.each do |jar|
67
- @logger.debug("JDBC - Loaded jar", :jar => jar)
68
- require jar
69
- end
70
-
71
- import @driver_class
71
+ load_jar_files!
72
72
 
73
- driver = Object.const_get(@driver_class[@driver_class.rindex('.') + 1, @driver_class.length]).new
74
- @connection = driver.connect(@connection_string, java.util.Properties.new)
73
+ @pool = Java::ComZaxxerHikari::HikariDataSource.new
74
+ @pool.setJdbcUrl(@connection_string)
75
+
76
+ @pool.setUsername(@username) if @username
77
+ @pool.setPassword(@password) if @password
75
78
 
76
- @logger.debug("JDBC - Created connection", :driver => driver, :connection => @connection)
79
+ @pool.setMaximumPoolSize(@max_pool_size)
80
+ @pool.setConnectionTimeout(@connection_timeout)
77
81
 
78
82
  if (@flush_size > 1000)
79
- @logger.warn("JDBC - Flush size is set to > 1000. May have performance penalties, depending on your SQL engine.")
83
+ @logger.warn("JDBC - Flush size is set to > 1000")
80
84
  end
81
85
 
82
86
  @repeat_exception_count = 0
@@ -101,7 +105,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
101
105
  end
102
106
 
103
107
  def flush(events, teardown=false)
104
- statement = @connection.prepareStatement(@statement[0])
108
+ connection = @pool.getConnection()
109
+
110
+ statement = connection.prepareStatement(@statement[0])
105
111
 
106
112
  events.each do |event|
107
113
  next if @statement.length < 2
@@ -132,6 +138,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
132
138
  begin
133
139
  @logger.debug("JDBC - Sending SQL", :sql => statement.toString())
134
140
  statement.executeBatch()
141
+ statement.close()
135
142
  rescue => e
136
143
  # Raising an exception will incur a retry from Stud::Buffer.
137
144
  # Since the exceutebatch failed this should mean any events failed to be
@@ -140,9 +147,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
140
147
  if e.getNextException() != nil
141
148
  @logger.warn("JDBC - Exception. Will automatically retry", :exception => e.getNextException())
142
149
  end
150
+ ensure
151
+ connection.close();
143
152
  end
144
-
145
- statement.close()
146
153
  end
147
154
 
148
155
  def on_flush_error(e)
@@ -165,8 +172,36 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
165
172
 
166
173
  def teardown
167
174
  buffer_flush(:final => true)
168
- @connection.close()
175
+ @pool.close()
169
176
  super
170
177
  end
171
178
 
179
+ private
180
+
181
+ def load_jar_files!
182
+ # Load jar from driver path
183
+ unless @driver_jar_path.nil?
184
+ raise Exception.new("JDBC - Could not find jar file at given path. Check config.") unless File.exists? @driver_jar_path
185
+ require @driver_jar_path
186
+ return
187
+ end
188
+
189
+ # Revert original behaviour of loading from vendor directory
190
+ # if no path given
191
+ if ENV['LOGSTASH_HOME']
192
+ jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar")
193
+ else
194
+ jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
195
+ end
196
+
197
+ @logger.debug("JDBC - jarpath", path: jarpath)
198
+
199
+ jars = Dir[jarpath]
200
+ raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty?
201
+
202
+ jars.each do |jar|
203
+ @logger.debug("JDBC - Loaded jar", :jar => jar)
204
+ require jar
205
+ end
206
+ end
172
207
  end # class LogStash::Outputs::jdbc
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-jdbc'
3
- s.version = "0.1.4"
3
+ s.version = "0.2.0.rc1"
4
4
  s.licenses = [ "Apache License (2.0)" ]
5
5
  s.summary = "This plugin allows you to output to SQL, via JDBC"
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
19
19
 
20
20
  # Gem dependencies
21
21
  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
22
+ s.add_runtime_dependency 'stud'
22
23
  s.add_runtime_dependency "logstash-codec-plain"
24
+
23
25
  s.add_development_dependency "logstash-devutils"
24
26
  end
metadata CHANGED
@@ -1,61 +1,75 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.2.0.rc1
5
5
  platform: ruby
6
6
  authors:
7
7
  - the_angry_angel
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-11-06 00:00:00.000000000 Z
11
+ date: 2015-11-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ">="
17
+ - - '>='
18
18
  - !ruby/object:Gem::Version
19
19
  version: 2.0.0.beta2
20
- - - "<"
20
+ - - <
21
21
  - !ruby/object:Gem::Version
22
22
  version: 3.0.0
23
23
  type: :runtime
24
24
  prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
- - - ">="
27
+ - - '>='
28
28
  - !ruby/object:Gem::Version
29
29
  version: 2.0.0.beta2
30
- - - "<"
30
+ - - <
31
31
  - !ruby/object:Gem::Version
32
32
  version: 3.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ name: stud
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - '>='
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
33
47
  - !ruby/object:Gem::Dependency
34
48
  name: logstash-codec-plain
35
49
  requirement: !ruby/object:Gem::Requirement
36
50
  requirements:
37
- - - ">="
51
+ - - '>='
38
52
  - !ruby/object:Gem::Version
39
53
  version: '0'
40
54
  type: :runtime
41
55
  prerelease: false
42
56
  version_requirements: !ruby/object:Gem::Requirement
43
57
  requirements:
44
- - - ">="
58
+ - - '>='
45
59
  - !ruby/object:Gem::Version
46
60
  version: '0'
47
61
  - !ruby/object:Gem::Dependency
48
62
  name: logstash-devutils
49
63
  requirement: !ruby/object:Gem::Requirement
50
64
  requirements:
51
- - - ">="
65
+ - - '>='
52
66
  - !ruby/object:Gem::Version
53
67
  version: '0'
54
68
  type: :development
55
69
  prerelease: false
56
70
  version_requirements: !ruby/object:Gem::Requirement
57
71
  requirements:
58
- - - ">="
72
+ - - '>='
59
73
  - !ruby/object:Gem::Version
60
74
  version: '0'
61
75
  description: This gem is a logstash plugin required to be installed on top of the
@@ -66,13 +80,17 @@ executables: []
66
80
  extensions: []
67
81
  extra_rdoc_files: []
68
82
  files:
69
- - ".gitignore"
83
+ - .gitignore
70
84
  - Gemfile
71
85
  - LICENSE.txt
72
86
  - README.md
73
87
  - Rakefile
88
+ - lib/logstash-output-jdbc_jars.rb
74
89
  - lib/logstash/outputs/jdbc.rb
75
90
  - logstash-output-jdbc.gemspec
91
+ - vendor/jar-dependencies/runtime-jars/HikariCP-2.4.2.jar
92
+ - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
93
+ - vendor/jar-dependencies/runtime-jars/slf4j-nop-1.7.13.jar
76
94
  homepage: https://github.com/theangryangel/logstash-output-jdbc
77
95
  licenses:
78
96
  - Apache License (2.0)
@@ -85,17 +103,17 @@ require_paths:
85
103
  - lib
86
104
  required_ruby_version: !ruby/object:Gem::Requirement
87
105
  requirements:
88
- - - ">="
106
+ - - '>='
89
107
  - !ruby/object:Gem::Version
90
108
  version: '0'
91
109
  required_rubygems_version: !ruby/object:Gem::Requirement
92
110
  requirements:
93
- - - ">="
111
+ - - '>'
94
112
  - !ruby/object:Gem::Version
95
- version: '0'
113
+ version: 1.3.1
96
114
  requirements: []
97
115
  rubyforge_project:
98
- rubygems_version: 2.4.5.1
116
+ rubygems_version: 2.0.14
99
117
  signing_key:
100
118
  specification_version: 4
101
119
  summary: This plugin allows you to output to SQL, via JDBC