logstash-output-jdbc 0.2.0.rc1 → 0.2.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4fb996001ccfc6faabb63f7d242ab88e176a8c1
-  data.tar.gz: e843a30f504af936835909f388dc7fe74c5e3d19
+  metadata.gz: e5d20a2e81aa5a05d7c63f205a9cf17ec4c694e8
+  data.tar.gz: 4fae148aa988fad70a15e62893e8a2a614ca13a3
 SHA512:
-  metadata.gz: 6da31e801e832f0e9150a468fc07e8126a457069b9536f1add378baf8121f146a17fb08e83be87a6c2cb667b27ed2812262eace113059030537d58655ce06c7c
-  data.tar.gz: d5a84a95b71c98a512c6bf66135c3f8b3e68260da9d777a0be2c0628e649f9a7b8fc1cfdb8febdd22a6180090cf807c153b74b45f3fb6445d5f9fb699fc0e447
+  metadata.gz: 0095e183425171cf0215fd8dd839ae28281a04664a8e70ae6330fbc77433f1641a3436e82a7cc8ef55d3d84cbb70244287b6f4e78f1b46d855052bda3fa38d6c
+  data.tar.gz: 546451f10d576dbcf585d9b27ba200a296039c2b4ea1f93bfe8431e96408cbd80504545d848d7227bb7588c3453a414022ee75b6b9a42ca1def85c847204be6f

data/lib/logstash/outputs/jdbc.rb

@@ -30,6 +30,12 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   # [ "insert into table (message) values(?)", "%{message}" ]
   config :statement, :validate => :array, :required => true
 
+  # If this is an unsafe statement, use event.sprintf
+  # This also has potential performance penalties due to having to create a 
+  # new statement for each event, rather than adding to the batch and issuing
+  # multiple inserts in 1 go
+  config :unsafe_statement, :validate => :boolean, :default => false
+
   # Number of connections in the pool to maintain
   config :max_pool_size, :validate => :number, :default => 5
 
@@ -72,7 +78,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
 
     @pool = Java::ComZaxxerHikari::HikariDataSource.new
     @pool.setJdbcUrl(@connection_string)
-    
+
     @pool.setUsername(@username) if @username
     @pool.setPassword(@password) if @password
 
@@ -105,50 +111,10 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   end
 
   def flush(events, teardown=false)
-    connection = @pool.getConnection()
-
-    statement = connection.prepareStatement(@statement[0])
-
-    events.each do |event|
-      next if @statement.length < 2
-
-      @statement[1..-1].each_with_index do |i, idx|
-        case event[i]
-        when Time, LogStash::Timestamp
-          # Most reliable solution, cross JDBC driver
-          statement.setString(idx + 1, event[i].iso8601())
-        when Fixnum, Integer
-          statement.setInt(idx + 1, event[i])
-        when Float
-          statement.setFloat(idx + 1, event[i])
-        when String
-          statement.setString(idx + 1, event[i])
-        when true
-          statement.setBoolean(idx + 1, true)
-        when false
-          statement.setBoolean(idx + 1, false)
-        else
-          statement.setString(idx + 1, event.sprintf(i))
-        end
-      end
-
-      statement.addBatch()
-    end
-
-    begin
-      @logger.debug("JDBC - Sending SQL", :sql => statement.toString())
-      statement.executeBatch()
-      statement.close()
-    rescue => e
-      # Raising an exception will incur a retry from Stud::Buffer.
-      # Since the exceutebatch failed this should mean any events failed to be
-      # inserted will be re-run. We're going to log it for the lols anyway.
-      @logger.warn("JDBC - Exception. Will automatically retry", :exception => e)
-      if e.getNextException() != nil
-	      @logger.warn("JDBC - Exception. Will automatically retry", :exception => e.getNextException())
-  	  end
-    ensure
-      connection.close();
+    if @unsafe_statement == true
+      unsafe_flush(events, teardown)
+    else
+      safe_flush(events, teardown)
     end
   end
 
@@ -204,4 +170,71 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
       require jar
     end
   end
+
+  def safe_flush(events, teardown=false)
+    connection = @pool.getConnection()
+
+    statement = connection.prepareStatement(@statement[0])
+
+    events.each do |event|
+      next if @statement.length < 2
+      statement = add_statement_event_params(statement, event)
+
+      statement.addBatch()
+    end
+
+    begin
+      @logger.debug("JDBC - Sending SQL", :sql => statement.toString())
+      statement.executeBatch()
+      statement.close()
+    rescue => e
+      # Raising an exception will incur a retry from Stud::Buffer.
+      # Since the exceutebatch failed this should mean any events failed to be
+      # inserted will be re-run. We're going to log it for the lols anyway.
+      @logger.warn("JDBC - Exception. Will automatically retry", :exception => e)
+      if e.getNextException() != nil
+        @logger.warn("JDBC - Exception. Will automatically retry", :exception => e.getNextException())
+      end
+    ensure
+      connection.close();
+    end
+  end
+
+  def unsafe_flush(events, teardown=false)
+    connection = @pool.getConnection()
+
+    events.each do |event|
+      statement = connection.prepareStatement(event.sprintf(@statement[0]))
+
+      statement = add_statement_event_params(statement, event) if @statement.length > 1
+
+      statement.execute()
+      statement.close()
+      connection.close()
+    end
+  end
+
+  def add_statement_event_params(statement, event)
+    @statement[1..-1].each_with_index do |i, idx|
+      case event[i]
+      when Time, LogStash::Timestamp
+        # Most reliable solution, cross JDBC driver
+        statement.setString(idx + 1, event[i].iso8601())
+      when Fixnum, Integer
+        statement.setInt(idx + 1, event[i])
+      when Float
+        statement.setFloat(idx + 1, event[i])
+      when String
+        statement.setString(idx + 1, event[i])
+      when true
+        statement.setBoolean(idx + 1, true)
+      when false
+        statement.setBoolean(idx + 1, false)
+      else
+        statement.setString(idx + 1, event.sprintf(i))
+      end
+    end
+
+    statement
+  end
 end # class LogStash::Outputs::jdbc

data/logstash-output-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-jdbc'
-  s.version = "0.2.0.rc1"
+  s.version = "0.2.0.rc2"
   s.licenses = [ "Apache License (2.0)" ]
   s.summary = "This plugin allows you to output to SQL, via JDBC"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.2.0.rc1
+  version: 0.2.0.rc2
 platform: ruby
 authors:
 - the_angry_angel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-14 00:00:00.000000000 Z
+date: 2015-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core