logstash-output-jdbc 0.2.0.rc3 → 0.2.0.rc4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a54974a84429e4dd7ae8f0b0ab08a2eca7ea0217
-  data.tar.gz: fde4c6acdf269342be353facc5caa8402656359c
+  metadata.gz: 1dc858c14f1596f4af6dd35a63139642c6a217c4
+  data.tar.gz: 3fc25dcb61a10107aef131240f4394ec4bf17814
 SHA512:
-  metadata.gz: e1e2792041f4caab27a0774cb8d63f3513bfe5add1debaeb1d08b634c806b21ff1f344d7c4c0905019e97fa2e5ce3ca92a0c0c590a4066bdc2e94aec1bc59a19
-  data.tar.gz: afc7f0e8a9027e098cd887ae96c4dd0625a8aac95d1ca4ce3bf49ec4aa2d0a16f9cde9feec3b1717ed09e317d0937f4a2ed3c3b6ac58bce6d9bd048af878b130
+  metadata.gz: c2d778ce352a48b759a314b9c138b6e7368091324659227cafe0ae4e2a4e5ee1b0cfe02b7046ff380094bbe9ea7af5d059898d9404417532324896d088e81a3f
+  data.tar.gz: cc1dee650cd5c497bf6fc0c0b01c2444fe1cfab211a89e212be018998f1d5a2f80a59f380d88b0de9252830e81073df51bc73313213d7e2c93c508f0bc935132

data/README.md

@@ -6,9 +6,16 @@ See below for tested adapters, and example configurations.
 
 This has not yet been extensively tested with all JDBC drivers and may not yet work for you.
 
+If you do find this works for a JDBC driver not listed, let me know and provide a small example configuration.
+
 This plugin does not bundle any JDBC jar files, and does expect them to be in a
 particular location. Please ensure you read the 4 installation lines below.
 
+## Headlines
+  - Support for connection pooling added in 0.2.0 [unreleased until #10 is resolved]
+  - Support for unsafe statement handling (allowing dynamic queries) in 0.2.0 [unreleased until #10 is resolved]
+  - Altered exception handling to now count sequential flushes with exceptions thrown in 0.2.0 [untested and unreleased until #10 is resolved]
+
 ## Versions
   - See master branch for logstash v2+
   - See v1.5 branch for logstash v1.5 
@@ -16,20 +23,32 @@ particular location. Please ensure you read the 4 installation lines below.
 
 ## Installation
   - Run `bin/plugin install logstash-output-jdbc` in your logstash installation directory
-  - Create the directory vendor/jar/jdbc in your logstash installation (`mkdir -p vendor/jar/jdbc/`)
-  - Add JDBC jar files to vendor/jar/jdbc in your logstash installation
-  - Configure
+  - Now either:
+    - Use driver_class in your configuraton to specify a path to your jar file
+  - Or:
+    - Create the directory vendor/jar/jdbc in your logstash installation (`mkdir -p vendor/jar/jdbc/`)
+    - Add JDBC jar files to vendor/jar/jdbc in your logstash installation
+  - And then configure (examples below)
 
 ## Configuration options
-  * driver_class, string, JDBC driver class to load
-  * connection_string, string, JDBC connection string
-  * statement, array, an array of strings representing the SQL statement to run. Index 0 is the SQL statement that is prepared, all other array entries are passed in as parameters (in order). A parameter may either be a property of the event (i.e. "@timestamp", or "host") or a formatted string (i.e. "%{host} - %{message}" or "%{message}"). If a key is passed then it will be automatically converted as required for insertion into SQL. If it's a formatted string then it will be passed in verbatim.
-  * flush_size, number, default = 1000, number of entries to buffer before sending to SQL
-  * idle_flush_time, number, default = 1, number of idle seconds before sending data to SQL, even if the flush_size has not been reached. If you modify this value you should also consider altering max_repeat_exceptions_time
-  * max_repeat_exceptions, number, default = 5, number of times the same exception can repeat before we stop logstash. Set to a value less than 1 if you never want it to stop
-  * max_repeat_exceptions_time, number, default = 30, maxium number of seconds between exceptions before they're considered "different" exceptions. If you modify idle_flush_time you should consider this value
+
+| Option | Type | Description | Required? | Default |
+| ------ | ---- | ----------- | --------- | ------- |
+| driver_path | String | File path to jar file containing your JDBC driver. This is optional, and all JDBC jars may be placed in $LOGSTASH_HOME/vendor/jar/jdbc instead. | No | |
+| connection_string | String | JDBC connection URL | Yes | |
+| username | String | JDBC username - this is optional as it may be included in the connection string, for many drivers | No | |
+| password | String | JDBC password - this is optional as it may be included in the connection string, for many drivers | No | |
+| statement | Array | An array of strings representing the SQL statement to run. Index 0 is the SQL statement that is prepared, all other array entries are passed in as parameters (in order). A parameter may either be a property of the event (i.e. "@timestamp", or "host") or a formatted string (i.e. "%{host} - %{message}" or "%{message}"). If a key is passed then it will be automatically converted as required for insertion into SQL. If it's a formatted string then it will be passed in verbatim. | Yes |  |
+| unsafe_statement | Boolean | If yes, the statement is evaluated for event fields - this allows you to use dynamic table names, etc. **This is highly dangerous** and you should **not** use this unless you are 100% sure that the field(s) you are passing in are 100% safe. Failure to do so will result in possible SQL injections. Please be aware that there is also a potential performance penalty as each event must be evaluated and inserted into SQL one at a time, where as when this is false multiple events are inserted at once. Example statement: [ "insert into %{table_name_field} (column) values(?)", "fieldname" ] | No | False |
+| max_pool_size | Number | Maximum number of connections to open to the SQL server at any 1 time | No | 5 |
+| connection_timeout | Number | Number of seconds before a SQL connection is closed | No | 2800 |
+| flush_size | Number | Maximum number of entries to buffer before sending to SQL - if this is reached before idle_flush_time | No | 1000 |
+| idle_flush_time | Number | Number of idle seconds before sending data to SQL - even if the flush_size has not yet been reached | No | 1 |
+| max_flush_exceptions | Number | Number of sequential flushes which cause an exception, before we stop logstash. Set to a value less than 1 if you never want it to stop. This should be carefully configured with relation to idle_flush_time if your SQL instance is not highly available. | No | 0 |
 
 ## Example configurations
+If you have a working sample configuration, for a DB thats not listed, pull requests are welcome.
+
 ### SQLite3
   * Tested using https://bitbucket.org/xerial/sqlite-jdbc
   * SQLite setup - `echo "CREATE table log (host text, timestamp datetime, message text);" | sqlite3 test.db`
@@ -42,7 +61,6 @@ output {
 	stdout { }
 
 	jdbc {
-		driver_class => 'org.sqlite.JDBC'
 		connection_string => 'jdbc:sqlite:test.db'
 		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ]
 	}
@@ -58,7 +76,6 @@ input
 }
 output {
 	jdbc {
-		driver_class => 'com.microsoft.sqlserver.jdbc.SQLServerDriver'
 		connection_string => "jdbc:sqlserver://server:1433;databaseName=databasename;user=username;password=password;autoReconnect=true;"
 		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ]
 	}
@@ -74,7 +91,6 @@ input
 }
 output {
 	jdbc {
-		driver_class => 'org.postgresql.Driver'
 		connection_string => 'jdbc:postgresql://hostname:5432/database?user=username&password=password'
 		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ]
 	}
@@ -92,7 +108,6 @@ input
 }
 output {
 	jdbc {
-		driver_class => "oracle.jdbc.driver.OracleDriver"
 		connection_string => "jdbc:oracle:thin:USER/PASS@HOST:PORT:SID"
 		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ]
 	}
@@ -110,9 +125,13 @@ input
 }
 output {
 	jdbc {
-		driver_class => "com.mysql.jdbc.Driver"
 		connection_string => "jdbc:mysql://HOSTNAME/DATABASE?user=USER&password=PASSWORD"
 		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ]
 	}
 }
 ```
+
+### MariaDB
+This is reportedly working, according to [@db2882](https://github.com/db2882) in issue #20. 
+No example configuration provided. 
+If you have a working sample, pull requests are welcome.

data/lib/logstash-output-jdbc_ring-buffer.rb

@@ -0,0 +1,19 @@
+class RingBuffer < Array
+  attr_reader :max_size
+
+  def initialize(max_size, enum = nil)
+    @max_size = max_size
+    enum.each { |e| self << e } if enum
+  end
+
+  def <<(el)
+    if self.size < @max_size || @max_size.nil?
+      super
+    else
+      self.shift
+      self.push(el)
+    end
+  end
+
+  alias :push :<<
+end

data/lib/logstash/outputs/jdbc.rb

@@ -4,6 +4,7 @@ require "logstash/namespace"
 require "stud/buffer"
 require "java"
 require "logstash-output-jdbc_jars"
+require "logstash-output-jdbc_ring-buffer"
 
 class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   # Adds buffer support
@@ -12,7 +13,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   config_name "jdbc"
 
   # Driver class - No longer required
-  config :driver_class, :obsolete => true
+  config :driver_class, :obsolete => "driver_class is no longer required and can be removed from your configuration"
 
   # Where to find the jar
   # Defaults to not required, and to the original behaviour
@@ -58,17 +59,20 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   # a timely manner.
   #
   # If you change this value please ensure that you change
-  # max_repeat_exceptions_time accordingly.
+  # max_flush_exceptions accordingly.
   config :idle_flush_time, :validate => :number, :default => 1
 
-  # Maximum number of repeating (sequential) exceptions, before we stop retrying
+  # Maximum number of sequential flushes which encounter exceptions, before we stop retrying.
   # If set to < 1, then it will infinitely retry.
-  config :max_repeat_exceptions, :validate => :number, :default => 4
+  # 
+  # You should carefully tune this in relation to idle_flush_time if your SQL server
+  # is not highly available.
+  # i.e. If your idle_flush_time is 1, and your max_flush_exceptions is 200, and your SQL server takes
+  # longer than 200 seconds to reboot, then logstash will stop.
+  config :max_flush_exceptions, :validate => :number, :default => 0
 
-  # The max number of seconds since the last exception, before we consider it
-  # a different cause.
-  # This value should be carefully considered in respect to idle_flush_time.
-  config :max_repeat_exceptions_time, :validate => :number, :default => 30
+  config :max_repeat_exceptions, :obsolete => "This has been replaced by max_flush_exceptions - which behaves slightly differently. Please check the documentation."
+  config :max_repeat_exceptions_time, :obsolete => "This is no longer required"
 
   public
   def register
@@ -85,17 +89,12 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     @pool.setMaximumPoolSize(@max_pool_size)
     @pool.setConnectionTimeout(@connection_timeout)
 
+    @exceptions_tracker = RingBuffer.new(@max_flush_exceptions)
+
     if (@flush_size > 1000)
       @logger.warn("JDBC - Flush size is set to > 1000")
     end
 
-    @repeat_exception_count = 0
-    @last_exception_time = Time.now
-
-    if (@max_repeat_exceptions > 0) and ((@idle_flush_time * @max_repeat_exceptions) > @max_repeat_exceptions_time)
-      @logger.warn("JDBC - max_repeat_exceptions_time is set such that it may still permit a looping exception. You probably changed idle_flush_time. Considering increasing max_repeat_exceptions_time.")
-    end
-
     buffer_initialize(
       :max_items => @flush_size,
       :max_interval => @idle_flush_time,
@@ -104,7 +103,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   end
 
   def receive(event)
-    return unless output?(event)
+    return unless output?(event) or event.cancelled?
     return unless @statement.length > 0
 
     buffer_receive(event)
@@ -119,21 +118,14 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   end
 
   def on_flush_error(e)
-    return if @max_repeat_exceptions < 1
+    return if @max_flush_exceptions < 1
 
-    if @last_exception == e.to_s
-      @repeat_exception_count += 1
-    else
-      @repeat_exception_count = 0
-    end
+    @exceptions_tracker << e.class
 
-    if (@repeat_exception_count >= @max_repeat_exceptions) and (Time.now - @last_exception_time) < @max_repeat_exceptions_time
-      @logger.error("JDBC - Exception repeated more than the maximum configured", :exception => e, :max_repeat_exceptions => @max_repeat_exceptions, :max_repeat_exceptions_time => @max_repeat_exceptions_time)
+    if @exceptions_tracker.reject { |i| i.nil? }.count >= @max_flush_exceptions
+      @logger.error("JDBC - max_flush_exceptions has been reached")
       raise e
     end
-
-    @last_exception_time = Time.now
-    @last_exception = e.to_s
   end
 
   def teardown
@@ -173,10 +165,10 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
 
   def safe_flush(events, teardown=false)
     connection = @pool.getConnection()
-
     statement = connection.prepareStatement(@statement[0])
 
     events.each do |event|
+      next if event.cancelled?
       next if @statement.length < 2
       statement = add_statement_event_params(statement, event)
 
@@ -184,17 +176,15 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     end
 
     begin
-      @logger.debug("JDBC - Sending SQL", :sql => statement.toString())
       statement.executeBatch()
       statement.close()
+      @exceptions_tracker << nil
+      
     rescue => e
       # Raising an exception will incur a retry from Stud::Buffer.
       # Since the exceutebatch failed this should mean any events failed to be
       # inserted will be re-run. We're going to log it for the lols anyway.
-      @logger.warn("JDBC - Exception. Will automatically retry", :exception => e)
-      if e.getNextException() != nil
-        @logger.warn("JDBC - Exception. Will automatically retry", :exception => e.getNextException())
-      end
+      log_jdbc_exception(e)
     ensure
       connection.close();
     end
@@ -204,13 +194,26 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     connection = @pool.getConnection()
 
     events.each do |event|
+      next if event.cancelled?
+      
       statement = connection.prepareStatement(event.sprintf(@statement[0]))
-
       statement = add_statement_event_params(statement, event) if @statement.length > 1
 
-      statement.execute()
-      statement.close()
-      connection.close()
+      begin
+        statement.execute()
+        
+        # cancel the event, since we may end up outputting the same event multiple times
+        # if an exception happens later down the line
+        event.cancel
+        @exceptions_tracker << nil
+      rescue => e
+        # Raising an exception will incur a retry from Stud::Buffer.
+        # We log for the lols.
+        log_jdbc_exception(e)
+      ensure
+        statement.close()
+        connection.close()
+      end
     end
   end
 
@@ -231,10 +234,23 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
       when false
         statement.setBoolean(idx + 1, false)
       else
-        statement.setString(idx + 1, event.sprintf(i))
+        if event[i].nil? and i =~ /%\{/
+          statement.setString(idx + 1, event.sprintf(i))
+        else
+          statement.setString(idx + 1, nil)
+        end
       end
     end
 
     statement
   end
+
+  def log_jdbc_exception(e)
+    ce = e
+    loop do
+      @logger.error("JDBC Exception encountered: Will automatically retry.", :exception => ce)
+      ce = e.getNextException()
+      break if ce == nil
+    end
+  end
 end # class LogStash::Outputs::jdbc

data/spec/outputs/jdbc_spec.rb

@@ -0,0 +1,13 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/jdbc"
+require "stud/temporary"
+
+describe LogStash::Outputs::Jdbc do
+
+  it "should register without errors" do
+    plugin = LogStash::Plugin.lookup("output", "jdbc").new({})
+    expect { plugin.register }.to_not raise_error
+    
+  end
+  
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.2.0.rc3
+  version: 0.2.0.rc4
 platform: ruby
 authors:
 - the_angry_angel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2015-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -82,9 +82,11 @@ extra_rdoc_files: []
 files:
 - lib/logstash/outputs/jdbc.rb
 - lib/logstash-output-jdbc_jars.rb
+- lib/logstash-output-jdbc_ring-buffer.rb
 - vendor/jar-dependencies/runtime-jars/HikariCP-2.4.2.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-nop-1.7.13.jar
+- spec/outputs/jdbc_spec.rb
 - LICENSE.txt
 - README.md
 homepage: https://github.com/theangryangel/logstash-output-jdbc
@@ -93,7 +95,8 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message: logstash-output-jdbc 0.2.0 introduces several new features -
+  please ensure you check the documentation in the README file
 rdoc_options: []
 require_paths:
 - lib
@@ -113,4 +116,5 @@ rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: This plugin allows you to output to SQL, via JDBC
-test_files: []
+test_files:
+- spec/outputs/jdbc_spec.rb