logstash-output-jdbc 0.3.2-java → 5.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe8c75828cf4838f6cb1df9ed10a067c82686398
-  data.tar.gz: 49d23e6f1ef4efb45cb6becf33af21a02d41b969
+  metadata.gz: 727a931f9b189e29d515b0798e010f276f5def1f
+  data.tar.gz: ed78ddac198f7eb2411da80c51893d44bd6fd52b
 SHA512:
-  metadata.gz: 726d2e0f254e004614bbe6f21ef9baccc77b382c6afbeffbf9dfd141a378dcb7b0349032c2a9d57b63f49051313cc0c750f2bcfa6432e53c7187602ac33e49b2
-  data.tar.gz: 20f0098bec13c4401560fdf6ba48391dc2607302a4055a1c5c3e98e50161d7811867fd9c2db7c5dbd3e8926054d74dd0db16629a51ae6418e13427a0b0fbd191
+  metadata.gz: 7176a744ec917960465f5c874eeb0f81ede3c89f442822f391b898326004ff92d274e6c08dd7312b26f224a72f353be1e91130200a722642b03fc1c96ca14061
+  data.tar.gz: f7597c9e0cf41ad3417ba26d316cfea0e0a9533f2e387f73578dc87e33e6041caedac209bb03e7fe05e2f9dfbf894d0cc749e626f7616ecd9309b866337f7125

data/README.md

@@ -1,6 +1,6 @@
 # logstash-output-jdbc 
 
-[![Build Status](https://travis-ci.org/theangryangel/logstash-output-jdbc.svg?branch=v2.x)](https://travis-ci.org/theangryangel/logstash-output-jdbc)
+[![Build Status](https://travis-ci.org/theangryangel/logstash-output-jdbc.svg?branch=master)](https://travis-ci.org/theangryangel/logstash-output-jdbc)
 
 This plugin is provided as an external plugin and is not part of the Logstash project.
 
@@ -21,7 +21,7 @@ See CHANGELOG.md
 Released versions are available via rubygems, and typically tagged.
 
 For development:
-  - See master branch for logstash v5 (currently **development only**)
+  - See master branch for logstash v5
   - See v2.x branch for logstash v2
   - See v1.5 branch for logstash v1.5 
   - See v1.4 branch for logstash 1.4
@@ -47,8 +47,8 @@ For development:
 | username | String | JDBC username - this is optional as it may be included in the connection string, for many drivers | No | |
 | password | String | JDBC password - this is optional as it may be included in the connection string, for many drivers | No | |
 | statement | Array | An array of strings representing the SQL statement to run. Index 0 is the SQL statement that is prepared, all other array entries are passed in as parameters (in order). A parameter may either be a property of the event (i.e. "@timestamp", or "host") or a formatted string (i.e. "%{host} - %{message}" or "%{message}"). If a key is passed then it will be automatically converted as required for insertion into SQL. If it's a formatted string then it will be passed in verbatim. | Yes |  |
-| unsafe_statement | Boolean | If yes, the statement is evaluated for event fields - this allows you to use dynamic table names, etc. **This is highly dangerous** and you should **not** use this unless you are 100% sure that the field(s) you are passing in are 100% safe. Failure to do so will result in possible SQL injections. Please be aware that there is also a potential performance penalty as each event must be evaluated and inserted into SQL one at a time, where as when this is false multiple events are inserted at once. Example statement: [ "insert into %{table_name_field} (column) values(?)", "fieldname" ] | No | False |
-| max_pool_size | Number | Maximum number of connections to open to the SQL server at any 1 time. Default set to same as Logstash default number of workers | No | 24 |
+| unsafe_statement | Boolean | If yes, the statement is evaluated for event fields - this allows you to use dynamic table names, etc. **This is highly dangerous** and you should **not** use this unless you are 100% sure that the field(s) you are passing in are 100% safe. Failure to do so will result in possible SQL injections. Example statement: [ "insert into %{table_name_field} (column) values(?)", "fieldname" ] | No | False |
+| max_pool_size | Number | Maximum number of connections to open to the SQL server at any 1 time | No | 5 |
 | connection_timeout | Number | Number of seconds before a SQL connection is closed | No | 2800 |
 | flush_size | Number | Maximum number of entries to buffer before sending to SQL - if this is reached before idle_flush_time | No | 1000 |
 | max_flush_exceptions | Number | Number of sequential flushes which cause an exception, before the set of events are discarded. Set to a value less than 1 if you never want it to stop. This should be carefully configured with respect to retry_initial_interval and retry_max_interval, if your SQL server is not highly available | No | 10 |

data/lib/logstash-output-jdbc_jars.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 require 'logstash/environment'
 
-root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, 'vendor')

data/lib/logstash/outputs/jdbc.rb

@@ -12,7 +12,7 @@ require 'logstash-output-jdbc_jars'
 # includes correctly crafting the SQL statement, and matching the number of
 # parameters correctly.
 class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
-  declare_threadsafe! if self.respond_to?(:declare_threadsafe!)
+  concurrency :shared
 
   STRFTIME_FMT = '%Y-%m-%d %T.%L'.freeze
 
@@ -98,7 +98,6 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   def register
     @logger.info('JDBC - Starting up')
 
-    LogStash::Logger.setup_log4j(@logger)
     load_jar_files!
 
     @stopping = Concurrent::AtomicBoolean.new(false)
@@ -122,10 +121,6 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     end
   end
 
-  def receive(event)
-    retrying_submit([event])
-  end
-
   def close
     @stopping.make_true
     @pool.close
@@ -156,7 +151,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     # Test connection
     test_connection = @pool.getConnection
     unless test_connection.isValid(validate_connection_timeout)
-      @logger.error('JDBC - Connection is not reporting as validate. Either connection is invalid, or driver is not getting the appropriate response.')
+      @logger.warn('JDBC - Connection is not reporting as validate. Either connection is invalid, or driver is not getting the appropriate response.')
     end
     test_connection.close
   end
@@ -177,13 +172,13 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
                 File.join(File.dirname(__FILE__), '../../../vendor/jar/jdbc/*.jar')
               end
 
-    @logger.debug('JDBC - jarpath', path: jarpath)
+    @logger.trace('JDBC - jarpath', path: jarpath)
 
     jars = Dir[jarpath]
     raise LogStash::ConfigurationError, 'JDBC - No jars found. Have you read the README?' if jars.empty?
 
     jars.each do |jar|
-      @logger.debug('JDBC - Loaded jar', jar: jar)
+      @logger.trace('JDBC - Loaded jar', jar: jar)
       require jar
     end
   end
@@ -258,14 +253,14 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
   def add_statement_event_params(statement, event)
     @statement[1..-1].each_with_index do |i, idx|
       if i.is_a? String 
-        value = event[i]
+        value = event.get(i)
         if value.nil? and i =~ /%\{/
           value = event.sprintf(i)
         end
       else
         value = i
       end
-      
+
       case value
       when Time
         # See LogStash::Timestamp, below, for the why behind strftime.
@@ -280,8 +275,6 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
         # strftime appears to be the most reliable across drivers.
         statement.setString(idx + 1, value.time.strftime(STRFTIME_FMT))
       when Fixnum, Integer
-        # bit_length doesn't exist in the current version of ruby/jruby logstash targets
-        # and this seems quicker than doing some Math.log2(value < 0 ? -value : value+1).ceil shit
         if value > 2147483647 or value < -2147483648
           statement.setLong(idx + 1, value)
         else
@@ -314,7 +307,7 @@ class LogStash::Outputs::Jdbc < LogStash::Outputs::Base
     log_method = (retrying ? 'warn' : 'error')
 
     loop do
-      @logger.send(log_method, log_text, :exception => current_exception, :backtrace => current_exception.backtrace)
+      @logger.send(log_method, log_text, :exception => current_exception)
 
       if current_exception.respond_to? 'getNextException'
         current_exception = current_exception.getNextException()

data/spec/jdbc_spec_helper.rb

@@ -4,6 +4,32 @@ require 'stud/temporary'
 require 'java'
 require 'securerandom'
 
+RSpec.configure do |c|
+
+  def start_service(name)
+    cmd = "sudo /etc/init.d/#{name}* start"
+
+    `which systemctl`
+    if $?.success?
+      cmd = "sudo systemctl start #{name}"
+    end
+
+    `#{cmd}`
+  end
+
+  def stop_service(name)
+    cmd = "sudo /etc/init.d/#{name}* stop"
+
+    `which systemctl`
+    if $?.success?
+      cmd = "sudo systemctl stop #{name}"
+    end
+
+    `#{cmd}`
+  end
+
+end
+
 RSpec.shared_context 'rspec setup' do
   it 'ensure jar is available' do
     expect(ENV[jdbc_jar_env]).not_to be_nil, "#{jdbc_jar_env} not defined, required to run tests"
@@ -20,7 +46,9 @@ RSpec.shared_context 'when initializing' do
 end
 
 RSpec.shared_context 'when outputting messages' do
-  let(:logger) { double("logger") }
+  let(:logger) { 
+    double("logger")
+  }
 
   let(:jdbc_test_table) do
     'logstash_output_jdbc_test'
@@ -43,16 +71,27 @@ RSpec.shared_context 'when outputting messages' do
   end
 
   let(:event_fields) do
-    { 'message' => "test-message #{SecureRandom.uuid}" }
+    { message: "test-message #{SecureRandom.uuid}" }
   end
 
   let(:event) { LogStash::Event.new(event_fields) }
 
   let(:plugin) do
+    # Setup logger
+    allow(LogStash::Outputs::Jdbc).to receive(:logger).and_return(logger)
+    
+    # XXX: Suppress reflection logging. There has to be a better way around this.
+    allow(logger).to receive(:debug).with(/config LogStash::/)
+
+    # Suppress beta warnings.
+    allow(logger).to receive(:info).with(/Please let us know if you find bugs or have suggestions on how to improve this plugin./)
+
+    # Suppress start up messages.
+    expect(logger).to receive(:info).once.with(/JDBC - Starting up/)
+
     # Setup plugin
     output = LogStash::Plugin.lookup('output', 'jdbc').new(jdbc_settings)
     output.register
-    output.logger = logger
 
     # Setup table
     c = output.instance_variable_get(:@pool).getConnection
@@ -82,7 +121,7 @@ RSpec.shared_context 'when outputting messages' do
     # Verify the number of items in the output table
     c = plugin.instance_variable_get(:@pool).getConnection
     stmt = c.prepareStatement("select count(*) as total from #{jdbc_test_table} where message = ?")
-    stmt.setString(1, event['message'])
+    stmt.setString(1, event.get('message'))
     rs = stmt.executeQuery
     count = 0
     count = rs.getInt('total') while rs.next
@@ -93,10 +132,14 @@ RSpec.shared_context 'when outputting messages' do
   end
 
   it 'should not save event, and log an unretryable exception' do
-    e = LogStash::Event.new({})
+    e = event
+    original_event = e.get('message')
+    e.set('message', nil)
 
     expect(logger).to receive(:error).once.with(/JDBC - Exception. Not retrying/, Hash)
-    expect { plugin.multi_receive([e]) }.to_not raise_error
+    expect { plugin.multi_receive([event]) }.to_not raise_error
+
+    e.set('message', original_event)
   end
 
   it 'it should retry after a connection loss, and log a warning' do
@@ -107,29 +150,21 @@ RSpec.shared_context 'when outputting messages' do
     # Check that everything is fine right now
     expect { p.multi_receive([event]) }.not_to raise_error
 
-    # Start a thread to stop and restart the service.
-    t = Thread.new(systemd_database_service) { |systemd_database_service|
-      start_stop_cmd = 'sudo /etc/init.d/%<service>s* %<action>s'
-
-      `which systemctl`
-      if $?.success?
-        start_stop_cmd = 'sudo systemctl %<action>s %<service>s'
-      end
-
-      cmd = start_stop_cmd % { action: 'stop', service: systemd_database_service }
-      `#{cmd}`
-      sleep 10
+    stop_service(systemd_database_service)
 
-      cmd = start_stop_cmd % { action: 'start', service: systemd_database_service }
-      `#{cmd}`
+    # Start a thread to restart the service after the fact.
+    t = Thread.new(systemd_database_service) { |systemd_database_service|
+      sleep 20
+    
+      start_service(systemd_database_service)
     }
 
-    # Wait a few seconds to the service to stop
-    sleep 5
-
+    t.run
+    
     expect(logger).to receive(:warn).at_least(:once).with(/JDBC - Exception. Retrying/, Hash)
     expect { p.multi_receive([event]) }.to_not raise_error
 
+    # Wait for the thread to finish
     t.join
   end
 end

metadata

@@ -1,160 +1,164 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 5.0.0
 platform: java
 authors:
 - the_angry_angel
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-15 00:00:00.000000000 Z
+date: 2016-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: stud
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-codec-plain
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: jar-dependencies
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: jar-dependencies
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-maven
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
-  type: :development
+  name: ruby-maven
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.41.2
-  type: :development
+  name: rubocop
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.41.2
-description: This gem is a logstash plugin required to be installed on top of the
-  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
-  a stand-alone program
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-jdbc'. This gem is not a stand-alone program
 email: karl+github@theangryangel.co.uk
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/logstash/outputs/jdbc.rb
+- LICENSE.txt
+- README.md
 - lib/logstash-output-jdbc_jars.rb
+- lib/logstash/outputs/jdbc.rb
 - spec/jdbc_spec_helper.rb
 - spec/outputs/jdbc_derby_spec.rb
 - spec/outputs/jdbc_mysql_spec.rb
 - spec/outputs/jdbc_spec.rb
 - spec/outputs/jdbc_sqlite_spec.rb
-- vendor/jar-dependencies/runtime-jars/HikariCP-2.4.2.jar
+- vendor/jar-dependencies/runtime-jars/HikariCP-2.4.7.jar
 - vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.12.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.21.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.21.jar
-- LICENSE.txt
-- README.md
 homepage: https://github.com/theangryangel/logstash-output-jdbc
 licenses:
 - Apache License (2.0)
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements:
-- jar 'com.zaxxer:HikariCP', '2.4.2'
+- jar 'com.zaxxer:HikariCP', '2.4.7'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.21'
-rubyforge_project: 
-rubygems_version: 2.0.14.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
 specification_version: 4
 summary: This plugin allows you to output to SQL, via JDBC
 test_files: