logstash-output-jdbc 0.2.10 → 5.4.0

data/logstash-output-jdbc.gemspec

@@ -0,0 +1,32 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-jdbc'
+  s.version = '5.4.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = 'This plugin allows you to output to SQL, via JDBC'
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-jdbc'. This gem is not a stand-alone program"
+  s.authors = ['the_angry_angel']
+  s.email = 'karl+github@theangryangel.co.uk'
+  s.homepage = 'https://github.com/theangryangel/logstash-output-jdbc'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
+
+  # Gem dependencies
+  #
+  s.add_runtime_dependency 'logstash-core-plugin-api', ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'logstash-devutils'
+
+  s.requirements << "jar 'com.zaxxer:HikariCP', '2.7.2'"
+  s.requirements << "jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.6.2'"
+
+  s.add_development_dependency 'jar-dependencies'
+  s.add_development_dependency 'ruby-maven', '~> 3.3'
+  s.add_development_dependency 'rubocop', '0.41.2'
+end

data/spec/jdbc_spec_helper.rb

@@ -0,0 +1,216 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/jdbc'
+require 'stud/temporary'
+require 'java'
+require 'securerandom'
+
+RSpec::Support::ObjectFormatter.default_instance.max_formatted_output_length = 80000
+
+RSpec.configure do |c|
+
+  def start_service(name)
+    cmd = "sudo /etc/init.d/#{name}* start"
+
+    `which systemctl`
+    if $?.success?
+      cmd = "sudo systemctl start #{name}"
+    end
+
+    `#{cmd}`
+  end
+
+  def stop_service(name)
+    cmd = "sudo /etc/init.d/#{name}* stop"
+
+    `which systemctl`
+    if $?.success?
+      cmd = "sudo systemctl stop #{name}"
+    end
+
+    `#{cmd}`
+  end
+
+end
+
+RSpec.shared_context 'rspec setup' do
+  it 'ensure jar is available' do
+    expect(ENV[jdbc_jar_env]).not_to be_nil, "#{jdbc_jar_env} not defined, required to run tests"
+    expect(File.exist?(ENV[jdbc_jar_env])).to eq(true), "#{jdbc_jar_env} defined, but not valid"
+  end
+end
+
+RSpec.shared_context 'when initializing' do
+  it 'shouldn\'t register with a missing jar file' do
+    jdbc_settings['driver_jar_path'] = nil
+    plugin = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
+    expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+  end
+end
+
+RSpec.shared_context 'when outputting messages' do
+  let(:logger) { 
+    double("logger")
+  }
+
+  let(:jdbc_test_table) do
+    'logstash_output_jdbc_test'
+  end
+
+  let(:jdbc_drop_table) do
+    "DROP TABLE #{jdbc_test_table}"
+  end
+
+  let(:jdbc_statement_fields) do
+    [
+      {db_field: "created_at",       db_type: "datetime",      db_value: '?',  event_field: '@timestamp'},
+      {db_field: "message",          db_type: "varchar(512)",  db_value: '?',  event_field: 'message'},
+      {db_field: "message_sprintf",  db_type: "varchar(512)",  db_value: '?',  event_field: 'sprintf-%{message}'},
+      {db_field: "static_int",       db_type: "int",           db_value: '?',  event_field: 'int'},
+      {db_field: "static_bigint",    db_type: "bigint",        db_value: '?',  event_field: 'bigint'},
+      {db_field: "static_float",     db_type: "float",         db_value: '?',  event_field: 'float'},
+      {db_field: "static_bool",      db_type: "boolean",       db_value: '?',  event_field: 'bool'},
+      {db_field: "static_bigdec",    db_type: "decimal",       db_value: '?',  event_field: 'bigdec'}
+    ]
+  end
+
+  let(:jdbc_create_table) do
+    fields = jdbc_statement_fields.collect { |entry| "#{entry[:db_field]} #{entry[:db_type]} not null" }.join(", ")
+
+    "CREATE table #{jdbc_test_table} (#{fields})"
+  end
+
+  let(:jdbc_drop_table) do
+    "DROP table #{jdbc_test_table}"
+  end
+
+  let(:jdbc_statement) do
+    fields = jdbc_statement_fields.collect { |entry| "#{entry[:db_field]}" }.join(", ")
+    values = jdbc_statement_fields.collect { |entry| "#{entry[:db_value]}" }.join(", ")
+    statement = jdbc_statement_fields.collect { |entry| entry[:event_field] }
+
+    statement.insert(0, "insert into #{jdbc_test_table} (#{fields}) values(#{values})")
+  end
+
+  let(:systemd_database_service) do
+    nil
+  end
+
+  let(:event) do
+    # TODO: Auto generate fields from jdbc_statement_fields
+    LogStash::Event.new({ 
+      message: "test-message #{SecureRandom.uuid}",
+      float: 12.1,
+      bigint: 4000881632477184,
+      bool: true,
+      int: 1,
+      bigdec: BigDecimal.new("123.123")
+    })
+  end
+
+  let(:plugin) do
+    # Setup logger
+    allow(LogStash::Outputs::Jdbc).to receive(:logger).and_return(logger)
+
+    # XXX: Suppress reflection logging. There has to be a better way around this.
+    allow(logger).to receive(:debug).with(/config LogStash::/)
+
+    # Suppress beta warnings.
+    allow(logger).to receive(:info).with(/Please let us know if you find bugs or have suggestions on how to improve this plugin./)
+
+    # Suppress start up messages.
+    expect(logger).to receive(:info).once.with(/JDBC - Starting up/)
+
+    # Setup plugin
+    output = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
+    output.register
+
+    output
+  end
+
+  before :each do
+    # Setup table
+    c = plugin.instance_variable_get(:@pool).getConnection
+
+    # Derby doesn't support IF EXISTS. 
+    # Seems like the quickest solution. Bleurgh.
+    begin
+      stmt = c.createStatement
+      stmt.executeUpdate(jdbc_drop_table)
+    rescue
+      # noop
+    ensure
+      stmt.close
+
+      stmt = c.createStatement
+      stmt.executeUpdate(jdbc_create_table)
+      stmt.close
+      c.close
+    end
+  end
+
+  # Delete table after each
+  after :each do
+    c = plugin.instance_variable_get(:@pool).getConnection
+
+    stmt = c.createStatement
+    stmt.executeUpdate(jdbc_drop_table)
+    stmt.close
+    c.close
+  end
+
+  it 'should save a event' do
+    expect { plugin.multi_receive([event]) }.to_not raise_error
+
+    # Verify the number of items in the output table
+    c = plugin.instance_variable_get(:@pool).getConnection
+
+    # TODO replace this simple count with a check of the actual contents
+
+    stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?")
+    stmt.setString(1, event.get('message'))
+    rs = stmt.executeQuery
+    count = 0
+    count = rs.getInt('total') while rs.next
+    stmt.close
+    c.close
+
+    expect(count).to eq(1)
+  end
+
+  it 'should not save event, and log an unretryable exception' do
+    e = event
+    original_event = e.get('message')
+    e.set('message', nil)
+
+    expect(logger).to receive(:error).once.with(/JDBC - Exception. Not retrying/, Hash)
+    expect { plugin.multi_receive([event]) }.to_not raise_error
+
+    e.set('message', original_event)
+  end
+
+  it 'it should retry after a connection loss, and log a warning' do
+    skip "does not run as a service, or known issue with test" if systemd_database_service.nil?
+
+    p = plugin
+
+    # Check that everything is fine right now
+    expect { p.multi_receive([event]) }.not_to raise_error
+
+    stop_service(systemd_database_service)
+
+    # Start a thread to restart the service after the fact.
+    t = Thread.new(systemd_database_service) { |systemd_database_service|
+      sleep 20
+
+      start_service(systemd_database_service)
+    }
+
+    t.run
+
+    expect(logger).to receive(:warn).at_least(:once).with(/JDBC - Exception. Retrying/, Hash)
+    expect { p.multi_receive([event]) }.to_not raise_error
+
+    # Wait for the thread to finish
+    t.join
+  end
+end

data/spec/outputs/jdbc_derby_spec.rb

@@ -0,0 +1,33 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: derby', if: ENV['JDBC_DERBY_JAR'] do
+  include_context 'rspec setup'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_DERBY_JAR'
+  end
+
+  let(:jdbc_statement_fields) do
+    [
+      {db_field: "created_at",       db_type: "timestamp",     db_value: 'CAST(? as timestamp)',  event_field: '@timestamp'},
+      {db_field: "message",          db_type: "varchar(512)",  db_value: '?',                     event_field: 'message'},
+      {db_field: "message_sprintf",  db_type: "varchar(512)",  db_value: '?',                     event_field: 'sprintf-%{message}'},
+      {db_field: "static_int",       db_type: "int",           db_value: '?',                     event_field: 'int'},
+      {db_field: "static_bigint",    db_type: "bigint",        db_value: '?',                     event_field: 'bigint'},
+      {db_field: "static_float",     db_type: "float",         db_value: '?',                     event_field: 'float'},
+      {db_field: "static_bool",      db_type: "boolean",       db_value: '?',                     event_field: 'bool'},
+      {db_field: "static_bigdec",    db_type: "decimal",       db_value: '?',                     event_field: 'bigdec'}      
+    ]
+  end
+
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'org.apache.derby.jdbc.EmbeddedDriver',
+      'connection_string' => 'jdbc:derby:memory:testdb;create=true',
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end

data/spec/outputs/jdbc_mysql_spec.rb

@@ -0,0 +1,24 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: mysql', if: ENV['JDBC_MYSQL_JAR'] do
+  include_context 'rspec setup'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_MYSQL_JAR'
+  end
+
+  let(:systemd_database_service) do
+    'mysql'
+  end
+
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'com.mysql.jdbc.Driver',
+      'connection_string' => 'jdbc:mysql://localhost/logstash?user=logstash&password=logstash',
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end

data/spec/outputs/jdbc_postgres_spec.rb

@@ -0,0 +1,41 @@
+require_relative '../jdbc_spec_helper'
+
+describe 'logstash-output-jdbc: postgres', if: ENV['JDBC_POSTGRES_JAR'] do
+  include_context 'rspec setup'
+  include_context 'when outputting messages'
+
+  let(:jdbc_jar_env) do
+    'JDBC_POSTGRES_JAR'
+  end
+
+  # TODO: Postgres doesnt kill connections fast enough for the test to pass
+  # Investigate options.
+
+  #let(:systemd_database_service) do
+  #  'postgresql'
+  #end
+
+  let(:jdbc_statement_fields) do
+    [
+      {db_field: "created_at",       db_type: "timestamp",     db_value: 'CAST(? as timestamp)',  event_field: '@timestamp'},
+      {db_field: "message",          db_type: "varchar(512)",  db_value: '?',                     event_field: 'message'},
+      {db_field: "message_sprintf",  db_type: "varchar(512)",  db_value: '?',                     event_field: 'sprintf-%{message}'},
+      {db_field: "static_int",       db_type: "int",           db_value: '?',                     event_field: 'int'},
+      {db_field: "static_bigint",    db_type: "bigint",        db_value: '?',                     event_field: 'bigint'},
+      {db_field: "static_float",     db_type: "float",         db_value: '?',                     event_field: 'float'},
+      {db_field: "static_bool",      db_type: "boolean",       db_value: '?',                     event_field: 'bool'},
+      {db_field: "static_bigdec",    db_type: "decimal",       db_value: '?',                     event_field: 'bigdec'}
+
+    ]
+  end
+
+  let(:jdbc_settings) do
+    {
+      'driver_class' => 'org.postgresql.Driver',
+      'connection_string' => 'jdbc:postgresql://localhost/logstash?user=logstash&password=logstash',
+      'driver_jar_path' => ENV[jdbc_jar_env],
+      'statement' => jdbc_statement,
+      'max_flush_exceptions' => 1
+    }
+  end
+end

data/spec/outputs/jdbc_spec.rb

@@ -1,94 +1,11 @@
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/jdbc"
-require "stud/temporary"
-require "java"
+require_relative '../jdbc_spec_helper'
 
 describe LogStash::Outputs::Jdbc do
-
-  let(:derby_settings) do
-    { 
-      "driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
-      "connection_string" => "jdbc:derby:memory:testdb;create=true",
-      "driver_jar_path" => ENV['JDBC_DERBY_JAR'],
-      "statement" => [ "insert into log (created_at, message) values(?, ?)", "@timestamp" "message" ]
-    }
-  end
-
-  context 'rspec setup' do
-
-    it 'ensure derby is available' do
-      j = ENV['JDBC_DERBY_JAR']
-      expect(j).not_to be_nil, "JDBC_DERBY_JAR not defined, required to run tests"
-      expect(File.exists?(j)).to eq(true), "JDBC_DERBY_JAR defined, but not valid"
-    end
-    
-  end
-
   context 'when initializing' do
-
     it 'shouldn\'t register without a config' do
-      expect { 
-        LogStash::Plugin.lookup("output", "jdbc").new()
-      }.to raise_error(LogStash::ConfigurationError)
-    end
-
-    it 'shouldn\'t register with a missing jar file' do
-      derby_settings['driver_jar_path'] = nil
-      plugin = LogStash::Plugin.lookup("output", "jdbc").new(derby_settings)
-      expect { plugin.register }.to raise_error
+      expect do
+        LogStash::Plugin.lookup('output', 'jdbc').new
+      end.to raise_error(LogStash::ConfigurationError)
     end
-
-    it 'shouldn\'t register with a missing jar file' do
-      derby_settings['connection_string'] = nil
-      plugin = LogStash::Plugin.lookup("output", "jdbc").new(derby_settings)
-      expect { plugin.register }.to raise_error
-    end
-
   end
-
-  context 'when outputting messages' do
-
-    let(:event_fields) do
-      { message: 'test-message' }
-    end
-    let(:event) { LogStash::Event.new(event_fields) }
-    let(:plugin) {
-      # Setup plugin
-      output = LogStash::Plugin.lookup("output", "jdbc").new(derby_settings)
-      output.register
-      if ENV['JDBC_DEBUG'] == '1'
-        output.logger.subscribe(STDOUT)
-      end
-
-      # Setup table
-      c = output.instance_variable_get(:@pool).getConnection()
-      stmt = c.createStatement()
-      stmt.executeUpdate("CREATE table log (created_at timestamp, message varchar(512))")
-      stmt.close()
-      c.close()
-
-      output
-    }
-
-    it 'should save a event' do
-      expect { plugin.receive(event) }.to_not raise_error
-      
-      # Wait for 1 second, for the buffer to flush
-      sleep 1
-
-      c = plugin.instance_variable_get(:@pool).getConnection()
-      stmt = c.createStatement()
-      rs = stmt.executeQuery("select count(*) as total from log")
-      count = 0
-      while rs.next()
-        count = rs.getInt("total")
-      end
-      stmt.close()
-      c.close()
-
-      expect(count).to be > 0
-    end
-
-  end
-
 end