logstash-input-jdbc 4.1.3 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 53ec2495d5e7fd90405e7c0b80e1fbf325f819d0
-  data.tar.gz: 1792c0903a6cf5943b1f4930ed5d2a6b60a6a9d2
+  metadata.gz: 985a22e0b06671dd557cc71e2174b39c654b15ee
+  data.tar.gz: 47347232f18c67f7f313814cb9663fcca0436d52
 SHA512:
-  metadata.gz: 5113d77634809fcac2ee4e1f64c8a5424cfb2626081237b5dff36f85edb8c135c719e9deba8a1945f78cc9ac2fabce2bce33ca39f4bc0a70ee14cfad5fd78da3
-  data.tar.gz: c21241ebf26bb0ae9ac69e9c1e9d1c77e486a9a925a3227f10bbf2b7ea3a6b64f2498fb78b0c17179ba867b87b35e8c40bf4dd954903df61da8520870eaeca3e
+  metadata.gz: 6048835db426fc1aab07e4324b358cd39e5db309144b20a34a90372474a110543c681b98cc5563cc8bc1042277565ca1f26f1420e47bfdba8fe7f6dac9c68a89
+  data.tar.gz: 17751bc5af1d066c19eaa15d2ef81b84b6a7556b88a2a2371b358162ba68f52e6ae0430ae5835bbd61f386974482ce27dc6244e128092fd46094b8e7889de06b

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.2.0
+ - Automatically reconnect on connection issues
+ - Fix test failures
+ - Explicitly load identifier mangling from Sequel to prevent
+   warning logs
+
 ## 4.1.3
  - Fix part1 of #172, coerce SQL DATE to LS Timestamp
 

data/lib/logstash/inputs/jdbc.rb

@@ -62,7 +62,7 @@ require "yaml" # persistence
 # instruct the plugin to execute this input statement on the minute, every minute.
 #
 # [source,ruby]
-# ----------------------------------
+# ------------------------------------------------------------------------------
 # input {
 #   jdbc {
 #     jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar"
@@ -74,7 +74,7 @@ require "yaml" # persistence
 #     statement => "SELECT * from songs where artist = :favorite_artist"
 #   }
 # }
-# ----------------------------------
+# ------------------------------------------------------------------------------
 #
 # ==== Configuring SQL statement
 #
@@ -84,6 +84,21 @@ require "yaml" # persistence
 # The file option only supports one SQL statement. The plugin will only accept one of the options.
 # It cannot read a statement from a file as well as from the `statement` configuration parameter.
 #
+# ==== Configuring multiple SQL statements 
+#
+# Configuring multiple SQL statements is useful when there is a need to query and ingest data 
+# from different database tables or views. It is possible to define separate Logstash 
+# configuration files for each statement or to define multiple statements in a single configuration 
+# file. When using multiple statements in a single Logstash configuration file, each statement 
+# has to be defined as a separate jdbc input (including jdbc driver, connection string and other 
+# required parameters). 
+#
+# Please note that if any of the statements use the `sql_last_value` parameter (e.g. for 
+# ingesting only data changed since last run), each input should define its own 
+# `last_run_metadata_path` parameter. Failure to do so will result in undesired behaviour, as
+# all inputs will store their state to the same (default) metadata file, effectively 
+# overwriting each other's `sql_last_value`.
+#
 # ==== Predefined Parameters
 #
 # Some parameters are built-in and can be used from within your queries.
@@ -97,16 +112,16 @@ require "yaml" # persistence
 #
 # Example:
 # [source,ruby]
-# ----------------------------------
+# ---------------------------------------------------------------------------------------------------
 # input {
 #   jdbc {
 #     statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value"
 #     use_column_value => true
-#     tracking_column => id
+#     tracking_column => "id"
 #     # ... other configuration bits
 #   }
 # }
-# ----------------------------------
+# ---------------------------------------------------------------------------------------------------
 #
 class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   include LogStash::PluginMixins::Jdbc
@@ -121,9 +136,9 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   # For example:
   #
   # [source, ruby]
-  # ----------------------------------
+  # -----------------------------------------------
   # "SELECT * FROM MYTABLE WHERE id = :target_id"
-  # ----------------------------------
+  # -----------------------------------------------
   #
   # here, ":target_id" is a named parameter. You can configure named parameters
   # with the `parameters` setting.
@@ -172,7 +187,7 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   #
   # Example:
   # [source,ruby]
-  # ----------------------------------
+  # -------------------------------------------------------
   # input {
   #   jdbc {
   #     ...
@@ -180,6 +195,7 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   #     ...
   #   }
   # }
+  # -------------------------------------------------------
   # this will only convert column0 that has ISO-8859-1 as an original encoding.
   config :columns_charset, :validate => :hash, :default => {}
 

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -138,8 +138,8 @@ module LogStash::PluginMixins::Jdbc
     end
   end
 
-  public
-  def prepare_jdbc_connection
+  private
+  def open_jdbc_connection
     require "java"
     require "sequel"
     require "sequel/adapters/jdbc"
@@ -170,16 +170,27 @@ module LogStash::PluginMixins::Jdbc
     begin
       @database.test_connection
     rescue Sequel::DatabaseConnectionError => e
+      @logger.warn("Failed test_connection.")
+      @database.close_jdbc_connection
+
       #TODO return false and let the plugin raise a LogStash::ConfigurationError
       raise e
     end
+
     @database.sql_log_level = @sql_log_level.to_sym
     @database.logger = @logger
+
+    @database.extension :identifier_mangling
+
     if @lowercase_column_names
       @database.identifier_output_method = :downcase
     else
       @database.identifier_output_method = :to_s
     end
+  end
+
+  public
+  def prepare_jdbc_connection
     if @use_column_value
       case @tracking_column_type
         when "numeric"
@@ -195,6 +206,7 @@ module LogStash::PluginMixins::Jdbc
   public
   def close_jdbc_connection
     @database.disconnect if @database
+    @database = nil
   end
 
   public
@@ -202,6 +214,7 @@ module LogStash::PluginMixins::Jdbc
     success = false
     begin
       parameters = symbolized_params(parameters)
+      open_jdbc_connection if @database == nil
       query = @database[statement, parameters]
       sql_last_value = @use_column_value ? @sql_last_value : Time.now.utc
       @tracking_column_warning_sent = false
@@ -229,6 +242,9 @@ module LogStash::PluginMixins::Jdbc
       success = true
     rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError => e
       @logger.warn("Exception when executing JDBC query", :exception => e)
+      @logger.warn("Attempt reconnection.")
+      close_jdbc_connection()
+      open_jdbc_connection()
     else
       @sql_last_value = sql_last_value
     end

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '4.1.3'
+  s.version         = '4.2.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example input streams a string at a definable interval."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -59,6 +59,7 @@ describe LogStash::Inputs::Jdbc do
       mixin_settings['jdbc_driver_library'] = '/foo/bar,/bar/foo'
       expect(plugin).to receive(:load_drivers).with(['/foo/bar', '/bar/foo'])
       plugin.register
+      plugin.run(queue) # load when first run
       plugin.stop
     end
 
@@ -66,6 +67,7 @@ describe LogStash::Inputs::Jdbc do
       mixin_settings['jdbc_driver_library'] = '/foo/bar'
       expect(plugin).to receive(:load_drivers).with(['/foo/bar'])
       plugin.register
+      plugin.run(queue) # load when first run
       plugin.stop
     end
 
@@ -819,7 +821,10 @@ describe LogStash::Inputs::Jdbc do
     end
 
     it "should fail" do
-      expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      expect do
+        plugin.register
+        plugin.run(queue) # load when first run
+      end.to raise_error(LogStash::ConfigurationError)
     end
   end
 
@@ -837,6 +842,7 @@ describe LogStash::Inputs::Jdbc do
 
     it "should raise PoolTimeout error" do
       plugin.register
+      plugin.run(queue)
       db = plugin.instance_variable_get(:@database)
       expect(db.pool.instance_variable_get(:@timeout)).to eq(0)
       expect(db.pool.instance_variable_get(:@max_size)).to eq(1)
@@ -852,7 +858,10 @@ describe LogStash::Inputs::Jdbc do
     it "should log error message" do
       allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
       expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 1 times.")
-      expect { plugin.register }.to raise_error(Sequel::PoolTimeout)
+      expect do
+        plugin.register
+        plugin.run(queue)
+      end.to raise_error(Sequel::PoolTimeout)
     end
   end
 
@@ -930,7 +939,10 @@ describe LogStash::Inputs::Jdbc do
       allow(Sequel).to receive(:connect).and_raise(Sequel::PoolTimeout)
       expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Trying again.")
       expect(plugin.logger).to receive(:error).with("Failed to connect to database. 0 second timeout exceeded. Tried 2 times.")
-      expect { plugin.register }.to raise_error(Sequel::PoolTimeout)
+      expect do
+        plugin.register
+        plugin.run(queue)
+      end.to raise_error(Sequel::PoolTimeout)
     end
 
     it "should not fail when passed a non-positive value" do
@@ -967,11 +979,14 @@ describe LogStash::Inputs::Jdbc do
         "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
         "column2" => 3
       }
+      event = LogStash::Event.new(row)
       expect(LogStash::Event).to receive(:new) do |row|
         row.each do |k, v|
           next unless v.is_a?(String)
           expect(row[k].encoding).to eq(encoded_row[k].encoding)
         end
+
+        event
       end
       plugin.run(events)
     end
@@ -999,11 +1014,14 @@ describe LogStash::Inputs::Jdbc do
           "column1" => "bar",
           "column2" => 3
         }
+        event = LogStash::Event.new(row)
         expect(LogStash::Event).to receive(:new) do |row|
           row.each do |k, v|
             next unless v.is_a?(String)
             expect(row[k].encoding).to eq(encoded_row[k].encoding)
           end
+          
+          event
         end
         plugin.run(events)
       end
@@ -1034,11 +1052,14 @@ describe LogStash::Inputs::Jdbc do
           "column2" => 3,
           "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
         }
+        event = LogStash::Event.new(row)
         expect(LogStash::Event).to receive(:new) do |row|
           row.each do |k, v|
             next unless v.is_a?(String)
             expect(row[k].encoding).to eq(encoded_row[k].encoding)
           end
+
+          event
         end
         plugin.run(events)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 4.1.3
+  version: 4.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-20 00:00:00.000000000 Z
+date: 2017-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement