logstash-output-jdbc 0.3.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 35dd5805a0cdbb3e95762b7416d3d7f438af203b
+  data.tar.gz: 641c6421320eb3a3374600404f3c7b2c12759a3d
+SHA512:
+  metadata.gz: fdb52ceb04117f3a06f09960286305660a4b3349a5b81292aa9b72a7f32a42b487ebb7fa518ca62e61d55474cdb775ada7f35f9b0c05956efaae2f3ba4741d67
+  data.tar.gz: 390fe8864c37a0d27bb14fd29112e0b57230546ca2af0eeee31c5d6a07a83b40c9dcf7f1064878e0a70e1b7c0f5ed7c8648e931b9ecd93038eb5b01c6d55fdb0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,84 @@
+# logstash-output-jdbc 
+
+[![Build Status](https://travis-ci.org/theangryangel/logstash-output-jdbc.svg?branch=master)](https://travis-ci.org/theangryangel/logstash-output-jdbc)
+
+This plugin is provided as an external plugin and is not part of the Logstash project.
+
+This plugin allows you to output to SQL databases, using JDBC adapters.
+See below for tested adapters, and example configurations.
+
+This has not yet been extensively tested with all JDBC drivers and may not yet work for you.
+
+If you do find this works for a JDBC driver without an example, let me know and provide a small example configuration if you can.
+
+This plugin does not bundle any JDBC jar files, and does expect them to be in a
+particular location. Please ensure you read the 4 installation lines below.
+
+## Changelog
+See CHANGELOG.md
+
+## Versions
+Released versions are available via rubygems, and typically tagged.
+
+For development:
+  - See master branch for logstash v5 (currently **development only**)
+  - See v2.x branch for logstash v2
+  - See v1.5 branch for logstash v1.5 
+  - See v1.4 branch for logstash 1.4
+
+## Installation
+  - Run `bin/logstash-plugin install logstash-output-jdbc` in your logstash installation directory
+  - Now either:
+    - Use driver_jar_path in your configuraton to specify a path to your jar file
+  - Or:
+    - Create the directory vendor/jar/jdbc in your logstash installation (`mkdir -p vendor/jar/jdbc/`)
+    - Add JDBC jar files to vendor/jar/jdbc in your logstash installation
+  - And then configure (examples can be found in the examples directory)
+
+## Configuration options
+
+| Option | Type | Description | Required? | Default |
+| ------ | ---- | ----------- | --------- | ------- |
+| driver_class | String | Specify a driver class if autoloading fails | No | |
+| driver_auto_commit | Boolean | If the driver does not support auto commit, you should set this to false | No | True |
+| driver_jar_path | String | File path to jar file containing your JDBC driver. This is optional, and all JDBC jars may be placed in $LOGSTASH_HOME/vendor/jar/jdbc instead. | No | |
+| connection_string | String | JDBC connection URL | Yes | |
+| username | String | JDBC username - this is optional as it may be included in the connection string, for many drivers | No | |
+| password | String | JDBC password - this is optional as it may be included in the connection string, for many drivers | No | |
+| statement | Array | An array of strings representing the SQL statement to run. Index 0 is the SQL statement that is prepared, all other array entries are passed in as parameters (in order). A parameter may either be a property of the event (i.e. "@timestamp", or "host") or a formatted string (i.e. "%{host} - %{message}" or "%{message}"). If a key is passed then it will be automatically converted as required for insertion into SQL. If it's a formatted string then it will be passed in verbatim. | Yes |  |
+| unsafe_statement | Boolean | If yes, the statement is evaluated for event fields - this allows you to use dynamic table names, etc. **This is highly dangerous** and you should **not** use this unless you are 100% sure that the field(s) you are passing in are 100% safe. Failure to do so will result in possible SQL injections. Please be aware that there is also a potential performance penalty as each event must be evaluated and inserted into SQL one at a time, where as when this is false multiple events are inserted at once. Example statement: [ "insert into %{table_name_field} (column) values(?)", "fieldname" ] | No | False |
+| max_pool_size | Number | Maximum number of connections to open to the SQL server at any 1 time. Default set to same as Logstash default number of workers | No | 24 |
+| connection_timeout | Number | Number of seconds before a SQL connection is closed | No | 2800 |
+| flush_size | Number | Maximum number of entries to buffer before sending to SQL - if this is reached before idle_flush_time | No | 1000 |
+| max_flush_exceptions | Number | Number of sequential flushes which cause an exception, before the set of events are discarded. Set to a value less than 1 if you never want it to stop. This should be carefully configured with respect to retry_initial_interval and retry_max_interval, if your SQL server is not highly available | No | 10 |
+| retry_initial_interval | Number | Number of seconds before the initial retry in the event of a failure. On each failure it will be doubled until it reaches retry_max_interval | No | 2 |
+| retry_max_interval | Number | Maximum number of seconds between each retry | No | 128 |
+| retry_sql_states | Array of strings | An array of custom SQL state codes you wish to retry until `max_flush_exceptions`. Useful if you're using a JDBC driver which returns retry-able, but non-standard SQL state codes in it's exceptions. | No | [] |
+
+## Example configurations
+Example logstash configurations, can now be found in the examples directory. Where possible we try to link every configuration with a tested jar.
+
+If you have a working sample configuration, for a DB thats not listed, pull requests are welcome.
+
+## Development and Running tests
+For development tests are recommended to run inside a virtual machine (Vagrantfile is included in the repo), as it requires
+access to various database engines and could completely destroy any data in a live system.
+
+If you have vagrant available (this is temporary whilst I'm hacking on v5 support. I'll make this more streamlined later):
+  - `vagrant up`
+  - `vagrant ssh`
+  - `cd /vagrant`
+  - `gem install bundler`
+  - `cd /vagrant && bundle install && bundle exec rake vendor && bundle exec rake install_jars`
+  - `./scripts/travis-before_script.sh && source ./scripts/travis-variables.sh`
+  - `bundle exec rspec`
+
+## Releasing
+  - Update Changelog
+  - Bump version in gemspec
+  - Commit
+  - Create tag `git tag v<version-number-in-gemspec>`
+  - `bundle exec rake install_jars`
+  - `bundle exec rake pre_release_checks`
+  - `gem build logstash-output-jdbc.gemspec`
+  - `gem push`

data/lib/logstash-output-jdbc_jars.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require 'logstash/environment'
+
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")

data/lib/logstash/outputs/jdbc.rb

@@ -0,0 +1,322 @@
+# encoding: utf-8
+require 'logstash/outputs/base'
+require 'logstash/namespace'
+require 'concurrent'
+require 'stud/interval'
+require 'java'
+require 'logstash-output-jdbc_jars'
+
+# Write events to a SQL engine, using JDBC.
+#
+# It is upto the user of the plugin to correctly configure the plugin. This
+# includes correctly crafting the SQL statement, and matching the number of
+# parameters correctly.
+class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
+  declare_threadsafe! if self.respond_to?(:declare_threadsafe!)
+
+  STRFTIME_FMT = '%Y-%m-%d %T.%L'.freeze
+
+  RETRYABLE_SQLSTATE_CLASSES = [
+    # Classes of retryable SQLSTATE codes
+    # Not all in the class will be retryable. However, this is the best that 
+    # we've got right now.
+    # If a custom state code is required, set it in retry_sql_states.
+    '08', # Connection Exception
+    '24', # Invalid Cursor State (Maybe retry-able in some circumstances)
+    '25', # Invalid Transaction State 
+    '40', # Transaction Rollback 
+    '53', # Insufficient Resources
+    '54', # Program Limit Exceeded (MAYBE)
+    '55', # Object Not In Prerequisite State
+    '57', # Operator Intervention
+    '58', # System Error
+  ].freeze
+
+  config_name 'jdbc'
+
+  # Driver class - Reintroduced for https://github.com/theangryangel/logstash-output-jdbc/issues/26
+  config :driver_class, validate: :string
+
+  # Does the JDBC driver support autocommit?
+  config :driver_auto_commit, validate: :boolean, default: true, required: true
+
+  # Where to find the jar
+  # Defaults to not required, and to the original behaviour
+  config :driver_jar_path, validate: :string, required: false
+
+  # jdbc connection string
+  config :connection_string, validate: :string, required: true
+
+  # jdbc username - optional, maybe in the connection string
+  config :username, validate: :string, required: false
+
+  # jdbc password - optional, maybe in the connection string
+  config :password, validate: :string, required: false
+
+  # [ "insert into table (message) values(?)", "%{message}" ]
+  config :statement, validate: :array, required: true
+
+  # If this is an unsafe statement, use event.sprintf
+  # This also has potential performance penalties due to having to create a
+  # new statement for each event, rather than adding to the batch and issuing
+  # multiple inserts in 1 go
+  config :unsafe_statement, validate: :boolean, default: false
+
+  # Number of connections in the pool to maintain
+  config :max_pool_size, validate: :number, default: 24
+
+  # Connection timeout
+  config :connection_timeout, validate: :number, default: 10000
+
+  # We buffer a certain number of events before flushing that out to SQL.
+  # This setting controls how many events will be buffered before sending a
+  # batch of events.
+  config :flush_size, validate: :number, default: 1000
+
+  # Set initial interval in seconds between retries. Doubled on each retry up to `retry_max_interval`
+  config :retry_initial_interval, validate: :number, default: 2
+
+  # Maximum time between retries, in seconds
+  config :retry_max_interval, validate: :number, default: 128
+
+  # Any additional custom, retryable SQL state codes. 
+  # Suitable for configuring retryable custom JDBC SQL state codes.
+  config :retry_sql_states, validate: :array, default: []
+
+  # Maximum number of sequential failed attempts, before we stop retrying.
+  # If set to < 1, then it will infinitely retry.
+  # At the default values this is a little over 10 minutes
+  config :max_flush_exceptions, validate: :number, default: 10
+
+  config :max_repeat_exceptions, obsolete: 'This has been replaced by max_flush_exceptions - which behaves slightly differently. Please check the documentation.'
+  config :max_repeat_exceptions_time, obsolete: 'This is no longer required'
+  config :idle_flush_time, obsolete: 'No longer necessary under Logstash v5'
+
+  def register
+    @logger.info('JDBC - Starting up')
+
+    LogStash::Logger.setup_log4j(@logger)
+    load_jar_files!
+
+    @stopping = Concurrent::AtomicBoolean.new(false)
+
+    @logger.warn('JDBC - Flush size is set to > 1000') if @flush_size > 1000
+
+    if @statement.empty?
+      @logger.error('JDBC - No statement provided. Configuration error.')
+    end
+
+    if !@unsafe_statement && @statement.length < 2
+      @logger.error("JDBC - Statement has no parameters. No events will be inserted into SQL as you're not passing any event data. Likely configuration error.")
+    end
+
+    setup_and_test_pool!
+  end
+
+  def multi_receive(events)
+    events.each_slice(@flush_size) do |slice|
+      retrying_submit(slice)
+    end
+  end
+
+  def receive(event)
+    retrying_submit([event])
+  end
+
+  def close
+    @stopping.make_true
+    @pool.close
+    super
+  end
+
+  private
+
+  def setup_and_test_pool!
+    # Setup pool
+    @pool = Java::ComZaxxerHikari::HikariDataSource.new
+
+    @pool.setAutoCommit(@driver_auto_commit)
+    @pool.setDriverClassName(@driver_class) if @driver_class
+
+    @pool.setJdbcUrl(@connection_string)
+
+    @pool.setUsername(@username) if @username
+    @pool.setPassword(@password) if @password
+
+    @pool.setMaximumPoolSize(@max_pool_size)
+    @pool.setConnectionTimeout(@connection_timeout)
+
+    validate_connection_timeout = (@connection_timeout / 1000) / 2
+
+    # Test connection
+    test_connection = @pool.getConnection
+    unless test_connection.isValid(validate_connection_timeout)
+      @logger.error('JDBC - Connection is not valid. Please check connection string or that your JDBC endpoint is available.')
+    end
+    test_connection.close
+  end
+
+  def load_jar_files!
+    # Load jar from driver path
+    unless @driver_jar_path.nil?
+      raise LogStash::ConfigurationError, 'JDBC - Could not find jar file at given path. Check config.' unless File.exist? @driver_jar_path
+      require @driver_jar_path
+      return
+    end
+
+    # Revert original behaviour of loading from vendor directory
+    # if no path given
+    jarpath = if ENV['LOGSTASH_HOME']
+                File.join(ENV['LOGSTASH_HOME'], '/vendor/jar/jdbc/*.jar')
+              else
+                File.join(File.dirname(__FILE__), '../../../vendor/jar/jdbc/*.jar')
+              end
+
+    @logger.debug('JDBC - jarpath', path: jarpath)
+
+    jars = Dir[jarpath]
+    raise LogStash::ConfigurationError, 'JDBC - No jars found. Have you read the README?' if jars.empty?
+
+    jars.each do |jar|
+      @logger.debug('JDBC - Loaded jar', jar: jar)
+      require jar
+    end
+  end
+
+  def submit(events)
+    connection = nil
+    statement = nil
+    events_to_retry = []
+
+    begin
+      connection = @pool.getConnection
+    rescue => e
+      log_jdbc_exception(e, true)
+      # If a connection is not available, then the server has gone away
+      # We're not counting that towards our retry count.
+      return events, false
+    end
+
+    events.each do |event|
+      begin
+        statement = connection.prepareStatement(
+          (@unsafe_statement == true) ? event.sprintf(@statement[0]) : @statement[0]
+        )
+        statement = add_statement_event_params(statement, event) if @statement.length > 1
+        statement.execute
+      rescue => e
+        if retry_exception?(e)
+          events_to_retry.push(event)
+        end
+      ensure
+        statement.close unless statement.nil?
+      end
+    end
+
+    connection.close unless connection.nil?
+
+    return events_to_retry, true
+  end
+
+  def retrying_submit(actions)
+    # Initially we submit the full list of actions
+    submit_actions = actions
+    count_as_attempt = true
+
+    attempts = 1
+
+    sleep_interval = @retry_initial_interval
+    while @stopping.false? and (submit_actions and !submit_actions.empty?)
+      return if !submit_actions || submit_actions.empty? # If everything's a success we move along
+      # We retry whatever didn't succeed
+      submit_actions, count_as_attempt = submit(submit_actions)
+
+      # Everything was a success!
+      break if !submit_actions || submit_actions.empty?
+
+      if @max_flush_exceptions > 0 and count_as_attempt == true
+        attempts += 1
+
+        if attempts > @max_flush_exceptions
+          @logger.error("JDBC - max_flush_exceptions has been reached. #{submit_actions.length} events have been unable to be sent to SQL and are being dropped. See previously logged exceptions for details.")
+          break
+        end
+      end
+
+      # If we're retrying the action sleep for the recommended interval
+      # Double the interval for the next time through to achieve exponential backoff
+      Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+      sleep_interval = next_sleep_interval(sleep_interval)
+    end
+  end
+
+  def add_statement_event_params(statement, event)
+    @statement[1..-1].each_with_index do |i, idx|
+      if i.is_a? String 
+        value = event[i]
+        if value.nil? and i =~ /%\{/
+          value = event.sprintf(i)
+        end
+      else
+        value = i
+      end
+      
+      case value
+      when Time
+        # See LogStash::Timestamp, below, for the why behind strftime.
+        statement.setString(idx + 1, value.strftime(STRFTIME_FMT))
+      when LogStash::Timestamp
+        # XXX: Using setString as opposed to setTimestamp, because setTimestamp
+        # doesn't behave correctly in some drivers (Known: sqlite)
+        #
+        # Additionally this does not use `to_iso8601`, since some SQL databases
+        # choke on the 'T' in the string (Known: Derby).
+        #
+        # strftime appears to be the most reliable across drivers.
+        statement.setString(idx + 1, value.time.strftime(STRFTIME_FMT))
+      when Fixnum, Integer
+        statement.setInt(idx + 1, value)
+      when Float
+        statement.setFloat(idx + 1, value)
+      when String
+        statement.setString(idx + 1, value)
+      when true, false
+        statement.setBoolean(idx + 1, value)
+      else
+        statement.setString(idx + 1, nil)
+      end
+    end
+
+    statement
+  end
+
+  def retry_exception?(exception)
+    retrying = (exception.respond_to? 'getSQLState' and (RETRYABLE_SQLSTATE_CLASSES.include?(exception.getSQLState.to_s[0,2]) or @retry_sql_states.include?(exception.getSQLState)))
+    log_jdbc_exception(exception, retrying)
+
+    retrying
+  end
+
+  def log_jdbc_exception(exception, retrying)
+    current_exception = exception
+    log_text = 'JDBC - Exception. ' + (retrying ? 'Retrying' : 'Not retrying') + '.'
+    log_method = (retrying ? 'warn' : 'error')
+
+    loop do
+      @logger.send(log_method, log_text, :exception => current_exception, :backtrace => current_exception.backtrace)
+
+      if current_exception.respond_to? 'getNextException'
+        current_exception = current_exception.getNextException()
+      else
+        current_exception = nil
+      end
+
+      break if current_exception == nil
+    end
+  end
+
+  def next_sleep_interval(current_interval)
+    doubled = current_interval * 2
+    doubled > @retry_max_interval ? @retry_max_interval : doubled
+  end
+end # class LogStash::Outputs::jdbc

data/spec/jdbc_spec_helper.rb

@@ -0,0 +1,135 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/jdbc'
+require 'stud/temporary'
+require 'java'
+require 'securerandom'
+
+RSpec.shared_context 'rspec setup' do
+  it 'ensure jar is available' do
+    expect(ENV[jdbc_jar_env]).not_to be_nil, "#{jdbc_jar_env} not defined, required to run tests"
+    expect(File.exist?(ENV[jdbc_jar_env])).to eq(true), "#{jdbc_jar_env} defined, but not valid"
+  end
+end
+
+RSpec.shared_context 'when initializing' do
+  it 'shouldn\'t register with a missing jar file' do
+    jdbc_settings['driver_jar_path'] = nil
+    plugin = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
+    expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+  end
+end
+
+RSpec.shared_context 'when outputting messages' do
+  let(:logger) { double("logger") }
+
+  let(:jdbc_test_table) do
+    'logstash_output_jdbc_test'
+  end
+
+  let(:jdbc_drop_table) do
+    "DROP TABLE #{jdbc_test_table}"
+  end
+
+  let(:jdbc_create_table) do
+    "CREATE table #{jdbc_test_table} (created_at datetime not null, message varchar(512) not null, message_sprintf varchar(512) not null, static_int int not null, static_bit bit not null)"
+  end
+
+  let(:jdbc_statement) do
+    ["insert into #{jdbc_test_table} (created_at, message, message_sprintf, static_int, static_bit) values(?, ?, ?, ?, ?)", '@timestamp', 'message', 'sprintf-%{message}', 1, true]
+  end
+
+  let(:systemd_database_service) do
+    nil
+  end
+
+  let(:event_fields) do
+    { 'message' => "test-message #{SecureRandom.uuid}" }
+  end
+
+  let(:event) { LogStash::Event.new(event_fields) }
+
+  let(:plugin) do
+    # Setup plugin
+    output = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
+    output.register
+    output.logger = logger
+
+    # Setup table
+    c = output.instance_variable_get(:@pool).getConnection
+
+    # Derby doesn't support IF EXISTS. 
+    # Seems like the quickest solution. Bleurgh.
+    begin
+      stmt = c.createStatement
+      stmt.executeUpdate(jdbc_drop_table)
+    rescue
+      # noop
+    ensure
+      stmt.close
+
+      stmt = c.createStatement
+      stmt.executeUpdate(jdbc_create_table)
+      stmt.close
+      c.close
+    end
+
+    output
+  end
+
+  it 'should save a event' do
+    expect { plugin.multi_receive([event]) }.to_not raise_error
+
+    # Verify the number of items in the output table
+    c = plugin.instance_variable_get(:@pool).getConnection
+    stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?")
+    stmt.setString(1, event['message'])
+    rs = stmt.executeQuery
+    count = 0
+    count = rs.getInt('total') while rs.next
+    stmt.close
+    c.close
+
+    expect(count).to eq(1)
+  end
+
+  it 'should not save event, and log an unretryable exception' do
+    e = LogStash::Event.new({})
+
+    expect(logger).to receive(:error).once.with(/JDBC - Exception. Not retrying/, Hash)
+    expect { plugin.multi_receive([e]) }.to_not raise_error
+  end
+
+  it 'it should retry after a connection loss, and log a warning' do
+    skip "does not run as a service" if systemd_database_service.nil?
+
+    p = plugin
+
+    # Check that everything is fine right now
+    expect { p.multi_receive([event]) }.not_to raise_error
+
+    # Start a thread to stop and restart the service.
+    t = Thread.new(systemd_database_service) { |systemd_database_service|
+      start_stop_cmd = 'sudo /etc/init.d/%<service>s* %<action>s'
+
+      `which systemctl`
+      if $?.success?
+        start_stop_cmd = 'sudo systemctl %<action>s %<service>s'
+      end
+
+      cmd = start_stop_cmd % { action: 'stop', service: systemd_database_service }
+      `#{cmd}`
+      sleep 10
+
+      cmd = start_stop_cmd % { action: 'start', service: systemd_database_service }
+      `#{cmd}`
+    }
+
+    # Wait a few seconds to the service to stop
+    sleep 5
+
+    expect(logger).to receive(:warn).at_least(:once).with(/JDBC - Exception. Retrying/, Hash)
+    expect { p.multi_receive([event]) }.to_not raise_error
+
+    t.join
+  end
+end

data/spec/outputs/jdbc_derby_spec.rb

@@ -0,0 +1,25 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: derby', if: ENV['JDBC_DERBY_JAR'] do
+  include_context 'rspec setup'
+  include_context 'when initializing'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_DERBY_JAR'
+  end
+
+  let(:jdbc_create_table) do
+    "CREATE table #{jdbc_test_table} (created_at timestamp not null, message varchar(512) not null, message_sprintf varchar(512) not null, static_int int not null, static_bit boolean not null)"
+  end
+ 
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'org.apache.derby.jdbc.EmbeddedDriver',
+      'connection_string' => 'jdbc:derby:memory:testdb;create=true',
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end

data/spec/outputs/jdbc_mysql_spec.rb

@@ -0,0 +1,25 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: mysql', if: ENV['JDBC_MYSQL_JAR'] do
+  include_context 'rspec setup'
+  include_context 'when initializing'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_MYSQL_JAR'
+  end
+
+  let(:systemd_database_service) do
+    'mysql'
+  end
+
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'com.mysql.jdbc.Driver',
+      'connection_string' => 'jdbc:mysql://localhost/logstash_output_jdbc_test?user=root',
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end

data/spec/outputs/jdbc_spec.rb

@@ -0,0 +1,11 @@
+require_relative '../jdbc_spec_helper'
+
+describe LogStash::Outputs::Jdbc do
+  context 'when initializing' do
+    it 'shouldn\'t register without a config' do
+      expect do
+        LogStash::Plugin.lookup('output', 'jdbc').new
+      end.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+end

data/spec/outputs/jdbc_sqlite_spec.rb

@@ -0,0 +1,27 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: sqlite', if: ENV['JDBC_SQLITE_JAR'] do
+  JDBC_SQLITE_FILE = '/tmp/logstash_output_jdbc_test.db'.freeze
+
+  before(:context) do
+    File.delete(JDBC_SQLITE_FILE) if File.exist? JDBC_SQLITE_FILE
+  end
+
+  include_context 'rspec setup'
+  include_context 'when initializing'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_SQLITE_JAR'
+  end
+
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'org.sqlite.JDBC',
+      'connection_string' => "jdbc:sqlite:#{JDBC_SQLITE_FILE}",
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end

metadata

@@ -0,0 +1,165 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-jdbc
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: java
+authors:
+- the_angry_angel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-maven
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
+email: karl+github@theangryangel.co.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/logstash/outputs/jdbc.rb
+- lib/logstash-output-jdbc_jars.rb
+- spec/jdbc_spec_helper.rb
+- spec/outputs/jdbc_derby_spec.rb
+- spec/outputs/jdbc_mysql_spec.rb
+- spec/outputs/jdbc_spec.rb
+- spec/outputs/jdbc_sqlite_spec.rb
+- vendor/jar-dependencies/runtime-jars/HikariCP-2.4.2.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.12.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.21.jar
+- LICENSE.txt
+- README.md
+homepage: https://github.com/theangryangel/logstash-output-jdbc
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'com.zaxxer:HikariCP', '2.4.2'
+- jar 'org.slf4j:slf4j-log4j12', '1.7.21'
+rubyforge_project: 
+rubygems_version: 2.0.14.1
+signing_key: 
+specification_version: 4
+summary: This plugin allows you to output to SQL, via JDBC
+test_files:
+- spec/jdbc_spec_helper.rb
+- spec/outputs/jdbc_derby_spec.rb
+- spec/outputs/jdbc_mysql_spec.rb
+- spec/outputs/jdbc_spec.rb
+- spec/outputs/jdbc_sqlite_spec.rb