logstash-output-jdbc 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 033e6d7684e87ec8494733ad218b09ad81145442
+  data.tar.gz: 4b2502b6210e89621a9416a32666d5e511082fe1
+SHA512:
+  metadata.gz: 4584a4a7129bf1dcbcb30a2d7bb3b628aef7bacd7a9d8607768ffcb2f77f63ee3ee915ed72b80f378fd55efaa6d7dd48a7722003816790f64b85f61ac18985dc
+  data.tar.gz: a75c975b799472844afd9448e2f054ac8ce6d1b618ba77daae8a4ceb3acce7919c3ae912a6044594f4d608528e5a08255e163926ffa8a6f9b0f6fe3c88d389ed

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+Gemfile.lock
+Gemfile.bak
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# logstash-jdbc
+JDBC output plugin for Logstash.
+This plugin is provided as an external plugin and is not part of the Logstash project.
+
+## Warning
+The master branch is for 1.5, is currently incomplete and should NOT be used (yet).
+
+Please see the v1.4 branch for v1.4 of Logstash.
+
+This has not yet been extensively tested with all JDBC drivers and may not yet work for you.
+
+## Installation
+  - Copy lib directory contents into your logstash installation.
+  - Create the directory vendor/jar/jdbc in your logstash installation (`mkdir -p vendor/jar/jdbc/`)
+  - Add JDBC jar files to vendor/jar/jdbc in your logstash installation
+  - Configure
+
+## Configuration options
+  * driver_class, string, JDBC driver class to load
+  * connection_string, string, JDBC connection string
+  * statement, array, an array of strings representing the SQL statement to run. Index 0 is the SQL statement that is prepared, all other array entries are passed in as parameters (in order). A parameter may either be a property of the event (i.e. "@timestamp", or "host") or a formatted string (i.e. "%{host} - %{message}" or "%{message}"). If a key is passed then it will be automatically converted as required for insertion into SQL. If it's a formatted string then it will be passed in verbatim.
+  * flush_size, number, default = 1000, number of entries to buffer before sending to SQL
+  * idle_flush_time, number, default = 1, number of idle seconds before sending data to SQL, even if the flush_size has not been reached. If you modify this value you should also consider altering max_repeat_exceptions_time
+  * max_repeat_exceptions, number, default = 5, number of times the same exception can repeat before we stop logstash. Set to a value less than 1 if you never want it to stop
+  * max_repeat_exceptions_time, number, default = 30, maxium number of seconds between exceptions before they're considered "different" exceptions. If you modify idle_flush_time you should consider this value
+
+## Example configurations
+### SQLite3
+  * Tested using https://bitbucket.org/xerial/sqlite-jdbc
+  * SQLite setup - `echo "CREATE table log (host text, timestamp datetime, message text);" | sqlite3 test.db`
+```
+input
+{
+	stdin { }
+}
+output {
+	stdout { }
+
+	jdbc {
+		driver_class => 'org.sqlite.JDBC'
+		connection_string => 'jdbc:sqlite:test.db'
+		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ]
+	}
+}
+```
+
+### SQL Server
+  * Tested using http://msdn.microsoft.com/en-gb/sqlserver/aa937724.aspx
+```
+input
+{
+	stdin { }
+}
+output {
+	jdbc {
+		driver_class => 'com.microsoft.sqlserver.jdbc.SQLServerDriver'
+		connection_string => "jdbc:sqlserver://server:1433;databaseName=databasename;user=username;password=password;autoReconnect=true;"
+		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, ?, ?)", "host", "@timestamp", "message" ]
+	}
+}
+```
+
+### Postgres
+With thanks to [@roflmao](https://github.com/roflmao)
+```
+input
+{
+	stdin { }
+}
+output {
+	jdbc {
+		driver_class => 'org.postgresql.Driver'
+		connection_string => 'jdbc:postgresql://hostname:5432/database?user=username&password=password'
+		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ]
+	}
+}
+```
+
+### Oracle
+With thanks to [@josemazo](https://github.com/josemazo)
+  * Tested with Express Edition 11g Release 2
+  * Tested using http://www.oracle.com/technetwork/database/enterprise-edition/jdbc-112010-090769.html (ojdbc6.jar)
+```
+input
+{
+	stdin { }
+}
+output {
+	jdbc {
+		driver_class => "oracle.jdbc.driver.OracleDriver"
+		connection_string => "jdbc:oracle:thin:USER/PASS@HOST:PORT:SID"
+		statement => [ "INSERT INTO log (host, timestamp, message) VALUES(?, CAST (? AS timestamp), ?)", "host", "@timestamp", "message" ]
+	}
+}
+```
+
+/* vim: set ts=4 sw=4 tw=0 :*/

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/outputs/jdbc.rb

@@ -0,0 +1,165 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "stud/buffer"
+require "java"
+
+class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
+  # Adds buffer support
+  include Stud::Buffer
+
+  config_name "jdbc"
+
+  # Driver class
+  config :driver_class, :validate => :string
+
+  # connection string
+  config :connection_string, :validate => :string, :required => true
+
+  # [ "insert into table (message) values(?)", "%{message}" ]
+  config :statement, :validate => :array, :required => true
+
+  # We buffer a certain number of events before flushing that out to SQL.
+  # This setting controls how many events will be buffered before sending a
+  # batch of events.
+  config :flush_size, :validate => :number, :default => 1000
+
+  # The amount of time since last flush before a flush is forced.
+  #
+  # This setting helps ensure slow event rates don't get stuck in Logstash.
+  # For example, if your `flush_size` is 100, and you have received 10 events,
+  # and it has been more than `idle_flush_time` seconds since the last flush,
+  # Logstash will flush those 10 events automatically.
+  #
+  # This helps keep both fast and slow log streams moving along in
+  # a timely manner.
+  #
+  # If you change this value please ensure that you change
+  # max_repeat_exceptions_time accordingly.
+  config :idle_flush_time, :validate => :number, :default => 1
+
+  # Maximum number of repeating (sequential) exceptions, before we stop retrying
+  # If set to < 1, then it will infinitely retry.
+  config :max_repeat_exceptions, :validate => :number, :default => 5
+
+  # The max number of seconds since the last exception, before we consider it
+  # a different cause.
+  # This value should be carefully considered in respect to idle_flush_time.
+  config :max_repeat_exceptions_time, :validate => :number, :default => 30
+
+  public
+  def register
+
+    @logger.info("JDBC - Starting up")
+
+    if ENV['LOGSTASH_HOME']
+      jarpath = File.join(ENV['LOGSTASH_HOME'], "/vendor/jar/jdbc/*.jar")
+    else
+      jarpath = File.join(File.dirname(__FILE__), "../../../vendor/jar/jdbc/*.jar")
+    end
+
+    @logger.debug("JDBC - jarpath", path: jarpath)
+
+    jars = Dir[jarpath]
+    raise Exception.new("JDBC - No jars found in jarpath. Have you read the README?") if jars.empty?
+
+    jars.each do |jar|
+      @logger.debug("JDBC - Loaded jar", :jar => jar)
+      require jar
+    end
+
+    import @driver_class
+
+    driver = Object.const_get(@driver_class[@driver_class.rindex('.') + 1, @driver_class.length]).new
+    @connection = driver.connect(@connection_string, java.util.Properties.new)
+
+    @logger.debug("JDBC - Created connection", :driver => driver, :connection => @connection)
+
+    if (@flush_size > 1000)
+      @logger.warn("JDBC - Flush size is set to > 1000. May have performance penalties, depending on your SQL engine.")
+    end
+
+    @repeat_exception_count = 0
+    @last_exception_time = Time.now
+
+    if (@max_repeat_exceptions > 0) and ((@idle_flush_time * @max_repeat_exceptions) > @max_repeat_exceptions_time)
+      @logger.warn("JDBC - max_repeat_exceptions_time is set such that it may still permit a looping exception. You probably changed idle_flush_time. Considering increasing max_repeat_exceptions_time.")
+    end
+
+    buffer_initialize(
+      :max_items => @flush_size,
+      :max_interval => @idle_flush_time,
+      :logger => @logger
+    )
+  end
+
+  def receive(event)
+    return unless output?(event)
+    return unless @statement.length > 0
+
+    buffer_receive(event)
+  end
+
+  def flush(events, teardown=false)
+    statement = @connection.prepareStatement(@statement[0])
+
+    events.each do |event|
+      next if @statement.length < 2
+
+      @statement[1..-1].each_with_index do |i, idx|
+        case event[i]
+        when Time, LogStash::Timestamp
+          # Most reliable solution, cross JDBC driver
+          statement.setString(idx + 1, event[i].iso8601())
+        when Fixnum, Integer
+          statement.setInt(idx + 1, event[i])
+        when Float
+          statement.setFloat(idx + 1, event[i])
+        when String
+          statement.setString(idx + 1, event[i])
+        else
+          statement.setString(idx + 1, event.sprintf(i))
+        end
+      end
+
+      statement.addBatch()
+    end
+
+    begin
+      @logger.debug("JDBC - Sending SQL", :sql => statement.toString())
+      statement.executeBatch()
+    rescue => e
+      # Raising an exception will incur a retry from Stud::Buffer.
+      # Since the exceutebatch failed this should mean any events failed to be
+      # inserted will be re-run. We're going to log it for the lols anyway.
+      @logger.warn("JDBC - Exception. Will automatically retry", :exception => e)
+    end
+
+    statement.close()
+  end
+
+  def on_flush_error(e)
+    return if @max_repeat_exceptions < 1
+
+    if @last_exception == e.to_s
+      @repeat_exception_count += 1
+    else
+      @repeat_exception_count = 0
+    end
+
+    if (@repeat_exception_count >= @max_repeat_exceptions) and (Time.now - @last_exception_time) < @max_repeat_exceptions_time
+      @logger.error("JDBC - Exception repeated more than the maximum configured", :exception => e, :max_repeat_exceptions => @max_repeat_exceptions, :max_repeat_exceptions_time => @max_repeat_exceptions_time)
+      raise e
+    end
+
+    @last_exception_time = Time.now
+    @last_exception = e.to_s
+  end
+
+  def teardown
+    buffer_flush(:final => true)
+    @connection.close()
+    super
+  end
+
+end # class LogStash::Outputs::jdbc

data/logstash-output-jdbc.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-jdbc'
+  s.version = "0.1.0"
+  s.licenses = [ "Apache License (2.0)" ]
+  s.summary = "This plugin allows you to output to SQL, via JDBC"
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["the_angry_angel"]
+  s.email = "karl+github@theangryangel.co.uk"
+  s.homepage = "https://github.com/theangryangel/logstash-output-jdbc"
+  s.require_paths = [ "lib" ]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 2.0.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_development_dependency "logstash-devutils"
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-jdbc
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- the_angry_angel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
+email: karl+github@theangryangel.co.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/logstash/outputs/jdbc.rb
+- logstash-output-jdbc.gemspec
+homepage: https://github.com/theangryangel/logstash-output-jdbc
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: This plugin allows you to output to SQL, via JDBC
+test_files: []