logstash-output-jdbc 0.1.4 → 0.2.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ceaf64925ffd4d57cbf2fb0a44ef53c1898ae3a
-  data.tar.gz: 21d2b8dfad9c8ab3f8dbfe74617d0fcf7cad7ffa
+  metadata.gz: f4fb996001ccfc6faabb63f7d242ab88e176a8c1
+  data.tar.gz: e843a30f504af936835909f388dc7fe74c5e3d19
 SHA512:
-  metadata.gz: c6dda6cbf171dbe2cca8a6d63d9d8524a29fb420975a726e220620c38fc043c523d27ee33c37ce7ddf0521fa0eadff2adfa31e9ba2665238a28adc46a53ebb8c
-  data.tar.gz: 168b6505cbb66157300a60c3009a17946afc72b45c17f4572476e962e43c1b14042622b49d98c59d7eefc63be85323f091d99f320f096a6d8c3d9563519c0209
+  metadata.gz: 6da31e801e832f0e9150a468fc07e8126a457069b9536f1add378baf8121f146a17fb08e83be87a6c2cb667b27ed2812262eace113059030537d58655ce06c7c
+  data.tar.gz: d5a84a95b71c98a512c6bf66135c3f8b3e68260da9d777a0be2c0628e649f9a7b8fc1cfdb8febdd22a6180090cf807c153b74b45f3fb6445d5f9fb699fc0e447

data/.gitignore

@@ -2,4 +2,3 @@
 Gemfile.lock
 Gemfile.bak
 .bundle
-vendor

data/lib/logstash-output-jdbc_jars.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require 'logstash/environment'
+
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")

data/lib/logstash/outputs/jdbc.rb

@@ -3,6 +3,7 @@ require "logstash/outputs/base"
 require "logstash/namespace"
 require "stud/buffer"
 require "java"
+require "logstash-output-jdbc_jars"
 
 class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   # Adds buffer support
@@ -10,15 +11,31 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
 
   config_name "jdbc"
 
-  # Driver class
-  config :driver_class, :validate => :string
+  # Driver class - No longer required
+  config :driver_class, :obsolete => true
 
-  # connection string
+  # Where to find the jar
+  # Defaults to not required, and to the original behaviour
+  config :driver_jar_path, :validate => :string, :required => false
+
+  # jdbc connection string
   config :connection_string, :validate => :string, :required => true
 
+  # jdbc username - optional, maybe in the connection string
+  config :username, :validate => :string, :required => false
+
+  # jdbc password - optional, maybe in the connection string
+  config :password, :validate => :string, :required => false
+
   # [ "insert into table (message) values(?)", "%{message}" ]
   config :statement, :validate => :array, :required => true
 
+  # Number of connections in the pool to maintain
+  config :max_pool_size, :validate => :number, :default => 5
+
+  # Connection timeout
+  config :connection_timeout, :validate => :number, :default => 2800
+
   # We buffer a certain number of events before flushing that out to SQL.
   # This setting controls how many events will be buffered before sending a
   # batch of events.
@@ -40,7 +57,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
 
   # Maximum number of repeating (sequential) exceptions, before we stop retrying
   # If set to < 1, then it will infinitely retry.
-  config :max_repeat_exceptions, :validate => :number, :default => 5
+  config :max_repeat_exceptions, :validate => :number, :default => 4
 
   # The max number of seconds since the last exception, before we consider it
   # a different cause.
@@ -49,34 +66,21 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
 
   public
   def register
-
     @logger.info("JDBC - Starting up")
 
-    if ENV['LOGSTASH_HOME']
-      jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar")
-    else
-      jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
-    end
-
-    @logger.debug("JDBC - jarpath", path: jarpath)
-
-    jars = Dir[jarpath]
-    raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty?
-
-    jars.each do |jar|
-      @logger.debug("JDBC - Loaded jar", :jar => jar)
-      require jar
-    end
-
-    import @driver_class
+    load_jar_files!
 
-    driver = Object.const_get(@driver_class[@driver_class.rindex('.') + 1, @driver_class.length]).new
-    @connection = driver.connect(@connection_string, java.util.Properties.new)
+    @pool = Java::ComZaxxerHikari::HikariDataSource.new
+    @pool.setJdbcUrl(@connection_string)
+    
+    @pool.setUsername(@username) if @username
+    @pool.setPassword(@password) if @password
 
-    @logger.debug("JDBC - Created connection", :driver => driver, :connection => @connection)
+    @pool.setMaximumPoolSize(@max_pool_size)
+    @pool.setConnectionTimeout(@connection_timeout)
 
     if (@flush_size > 1000)
-      @logger.warn("JDBC - Flush size is set to > 1000. May have performance penalties, depending on your SQL engine.")
+      @logger.warn("JDBC - Flush size is set to > 1000")
     end
 
     @repeat_exception_count = 0
@@ -101,7 +105,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   end
 
   def flush(events, teardown=false)
-    statement = @connection.prepareStatement(@statement[0])
+    connection = @pool.getConnection()
+
+    statement = connection.prepareStatement(@statement[0])
 
     events.each do |event|
       next if @statement.length < 2
@@ -132,6 +138,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     begin
       @logger.debug("JDBC - Sending SQL", :sql => statement.toString())
       statement.executeBatch()
+      statement.close()
     rescue => e
       # Raising an exception will incur a retry from Stud::Buffer.
       # Since the exceutebatch failed this should mean any events failed to be
@@ -140,9 +147,9 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
       if e.getNextException() != nil
 	      @logger.warn("JDBC - Exception. Will automatically retry", :exception => e.getNextException())
   	  end
+    ensure
+      connection.close();
     end
-
-    statement.close()
   end
 
   def on_flush_error(e)
@@ -165,8 +172,36 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
 
   def teardown
     buffer_flush(:final => true)
-    @connection.close()
+    @pool.close()
     super
   end
 
+  private
+
+  def load_jar_files!
+    # Load jar from driver path
+    unless @driver_jar_path.nil?
+      raise Exception.new("JDBC - Could not find jar file at given path. Check config.") unless File.exists? @driver_jar_path
+      require @driver_jar_path
+      return
+    end
+
+    # Revert original behaviour of loading from vendor directory
+    # if no path given
+    if ENV['LOGSTASH_HOME']
+      jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar")
+    else
+      jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
+    end
+
+    @logger.debug("JDBC - jarpath", path: jarpath)
+
+    jars = Dir[jarpath]
+    raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty?
+
+    jars.each do |jar|
+      @logger.debug("JDBC - Loaded jar", :jar => jar)
+      require jar
+    end
+  end
 end # class LogStash::Outputs::jdbc

data/logstash-output-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-jdbc'
-  s.version = "0.1.4"
+  s.version = "0.2.0.rc1"
   s.licenses = [ "Apache License (2.0)" ]
   s.summary = "This plugin allows you to output to SQL, via JDBC"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,6 +19,8 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency 'stud'
   s.add_runtime_dependency "logstash-codec-plain"
+  
   s.add_development_dependency "logstash-devutils"
 end

metadata

@@ -1,61 +1,75 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0.rc1
 platform: ruby
 authors:
 - the_angry_angel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-06 00:00:00.000000000 Z
+date: 2015-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.0.0.beta2
-    - - "<"
+    - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 2.0.0.beta2
-    - - "<"
+    - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the
@@ -66,13 +80,17 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
+- .gitignore
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- lib/logstash-output-jdbc_jars.rb
 - lib/logstash/outputs/jdbc.rb
 - logstash-output-jdbc.gemspec
+- vendor/jar-dependencies/runtime-jars/HikariCP-2.4.2.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-nop-1.7.13.jar
 homepage: https://github.com/theangryangel/logstash-output-jdbc
 licenses:
 - Apache License (2.0)
@@ -85,17 +103,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: This plugin allows you to output to SQL, via JDBC