logstash-input-jdbc 2.0.5 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6fcc8b3eaba96543efed3777360fd0af48db6bae
-  data.tar.gz: 6c87821ec96f4df83b055f5510b03ec2e5bba48f
+  metadata.gz: f9bab690eb90f270679aa61551938fa2bf7ee6de
+  data.tar.gz: ef01a9593412b885252102358fde81cb50a268e2
 SHA512:
-  metadata.gz: 38f419423f8fe7d82988049061049b6ff1af9c099e6af124da46aa4d1957076382c6fef6b49f461e83ffe9c562fe0f7e3df3ddeda08c9e786cf2057fda2aede2
-  data.tar.gz: 1d068d7aa44690298e49fe8ce926bc67c1663970cb10418d8d27f4879a8932dbc7c5027f84a295222906f90a8aa799b4f75838fa76dbed6e79ecc8dc1bca1885
+  metadata.gz: bfb164e27f19f96e41026ceaca231a62855aef0e1d9c4e38ab39f923d9d0059bb39361576a442c78fa6b9f618acb6df33dd196b581337b53fc3f26ac6b4f2d64
+  data.tar.gz: 854657f5cecda124c91baf0342921b7cbca9e451957a33587cb670f189cb7dbf93312d3cfdd28634509d3a1858baeedc6f9a4c6a9827cb9fbc8a9f8ba3b08eb6

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.1.0
+  - [#85](https://github.com/logstash-plugins/logstash-input-jdbc/issues/85) make the jdbc_driver_library accept a list of elements separated by commas as in some situations we might need to load more than one jar/lib.
+  - [#89](https://github.com/logstash-plugins/logstash-input-jdbc/issues/89) Set application timezone for cases where time fields in data have no timezone.
+
 ## 2.0.5
   - [#77](https://github.com/logstash-plugins/logstash-input-jdbc/issues/77) Time represented as RubyTime and not as Logstash::Timestamp
 
@@ -10,7 +14,7 @@
  - Added catch-all configuration option for any other options that Sequel lib supports
 
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -1,8 +1,10 @@
 # encoding: utf-8
 # TAKEN FROM WIIBAA
 require "logstash/config/mixin"
+require "time"
+require "date"
 
-# Tentative of abstracting JDBC logic to a mixin 
+# Tentative of abstracting JDBC logic to a mixin
 # for potential reuse in other plugins (input/output)
 module LogStash::PluginMixins::Jdbc
 
@@ -18,10 +20,11 @@ module LogStash::PluginMixins::Jdbc
 
   public
   def setup_jdbc_config
-    # JDBC driver library path to third party driver library.
+    # JDBC driver library path to third party driver library. In case of multiple libraries being
+    # required you can pass them separated by a comma.
     #
     # If not provided, Plugin will look for the driver class in the Logstash Java classpath.
-    config :jdbc_driver_library, :validate => :path
+    config :jdbc_driver_library, :validate => :string
 
     # JDBC driver class to load, for exmaple, "org.apache.derby.jdbc.ClientDriver"
     # NB per https://github.com/logstash-plugins/logstash-input-jdbc/issues/43 if you are using
@@ -44,7 +47,7 @@ module LogStash::PluginMixins::Jdbc
     # result-set. The limit size is set with `jdbc_page_size`.
     #
     # Be aware that ordering is not guaranteed between queries.
-    config :jdbc_paging_enabled, :validate => :boolean, :default => false 
+    config :jdbc_paging_enabled, :validate => :boolean, :default => false
 
     # JDBC page size
     config :jdbc_page_size, :validate => :number, :default => 100000
@@ -64,6 +67,15 @@ module LogStash::PluginMixins::Jdbc
     # The amount of seconds to wait to acquire a connection before raising a PoolTimeoutError (default 5)
     config :jdbc_pool_timeout, :validate => :number, :default => 5
 
+    # Timezone conversion.
+    # SQL does not allow for timezone data in timestamp fields.  This plugin will automatically
+    # convert your SQL timestamp fields to Logstash timestamps, in relative UTC time in ISO8601 format.
+    #
+    # Using this setting will manually assign a specified timezone offset, instead
+    # of using the timezone setting of the local machine.  You must use a canonical
+    # timezone, *America/Denver*, for example.
+    config :jdbc_default_timezone, :validate => :string
+
     # General/Vendor-specific Sequel configuration options.
     #
     # An example of an optional connection pool configuration
@@ -72,6 +84,10 @@ module LogStash::PluginMixins::Jdbc
     # examples of vendor-specific options can be found in this
     # documentation page: https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc
     config :sequel_opts, :validate => :hash, :default => {}
+
+    # Log level at which to log SQL queries, the accepted values are the common ones fatal, error, warn,
+    # info and debug. The default value is info.
+    config :sql_log_level, :validate => [ "fatal", "error", "warn", "info", "debug" ], :default => "info"
   end
 
   private
@@ -92,17 +108,29 @@ module LogStash::PluginMixins::Jdbc
     end
   end
 
+  private
+  def load_drivers(drivers)
+    drivers.each do |driver|
+      begin
+        require driver
+      rescue => e
+        @logger.error("Failed to load #{driver}", :exception => e)
+      end
+    end
+  end
+
   public
   def prepare_jdbc_connection
     require "java"
     require "sequel"
     require "sequel/adapters/jdbc"
-    require @jdbc_driver_library if @jdbc_driver_library
+    load_drivers(@jdbc_driver_library.split(",")) if @jdbc_driver_library
+
     begin
       Sequel::JDBC.load_driver(@jdbc_driver_class)
     rescue Sequel::AdapterNotFound => e
       message = if @jdbc_driver_library.nil?
-                  ":jdbc_driver_library is not set, are you sure you included 
+                  ":jdbc_driver_library is not set, are you sure you included
                   the proper driver client libraries in your classpath?"
                 else
                   "Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
@@ -111,6 +139,10 @@ module LogStash::PluginMixins::Jdbc
     end
     @database = jdbc_connect()
     @database.extension(:pagination)
+    if @jdbc_default_timezone
+      @database.extension(:named_timezones)
+      @database.timezone = @jdbc_default_timezone
+    end
     if @jdbc_validate_connection
       @database.extension(:connection_validator)
       @database.pool.connection_validation_timeout = @jdbc_validation_timeout
@@ -122,7 +154,8 @@ module LogStash::PluginMixins::Jdbc
       #TODO return false and let the plugin raise a LogStash::ConfigurationError
       raise e
     end
-
+    @database.sql_log_level = @sql_log_level.to_sym
+    @database.logger = @logger
     @sql_last_start = Time.at(0).utc
   end # def prepare_jdbc_connection
 
@@ -134,10 +167,10 @@ module LogStash::PluginMixins::Jdbc
   public
   def execute_statement(statement, parameters)
     success = false
-    begin 
+    begin
       parameters = symbolized_params(parameters)
       query = @database[statement, parameters]
-      @logger.debug? and @logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters)
+      @logger.debug? and @logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters, :count => query.count)
       @sql_last_start = Time.now.utc
 
       if @jdbc_paging_enabled
@@ -159,9 +192,9 @@ module LogStash::PluginMixins::Jdbc
   end
 
   # Symbolize parameters keys to use with Sequel
-  private 
+  private
   def symbolized_params(parameters)
-    parameters.inject({}) do |hash,(k,v)| 
+    parameters.inject({}) do |hash,(k,v)|
       case v
       when LogStash::Timestamp
         hash[k.to_sym] = v.time
@@ -180,9 +213,14 @@ module LogStash::PluginMixins::Jdbc
 
   private
   def decorate_value(value)
+
     if value.is_a?(Time)
       # transform it to LogStash::Timestamp as required by LS
       LogStash::Timestamp.new(value)
+    elsif value.is_a?(DateTime)
+      # Manual timezone conversion detected.
+      # This is slower, so we put it in as a conditional case.
+      LogStash::Timestamp.new(Time.parse(value.to_s))
     else
       value  # no-op
     end

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '2.0.5'
+  s.version         = '2.1.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example input streams a string at a definable interval."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -21,6 +21,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'sequel'
+  s.add_runtime_dependency 'tzinfo'
+  s.add_runtime_dependency 'tzinfo-data'
   s.add_runtime_dependency 'rufus-scheduler'
 
   s.add_development_dependency 'logstash-devutils'

data/spec/inputs/jdbc_spec.rb

@@ -5,9 +5,18 @@ require "sequel"
 require "sequel/adapters/jdbc"
 require "timecop"
 require "stud/temporary"
+require "time"
+require "date"
 
 describe LogStash::Inputs::Jdbc do
-  let(:mixin_settings) { {"jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver", "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"} }
+  # This is a necessary change test-wide to guarantee that no local timezone
+  # is picked up.  It could be arbitrarily set to any timezone, but then the test
+  # would have to compensate differently.  That's why UTC is chosen.
+  ENV["TZ"] = "Etc/UTC"
+  let(:mixin_settings) do
+    { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+      "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"}
+  end
   let(:settings) { {} }
   let(:plugin) { LogStash::Inputs::Jdbc.new(mixin_settings.merge(settings)) }
   let(:queue) { Queue.new }
@@ -42,6 +51,20 @@ describe LogStash::Inputs::Jdbc do
       plugin.stop
     end
 
+    it "should load all drivers when passing an array" do
+      mixin_settings['jdbc_driver_library'] = '/foo/bar,/bar/foo'
+      expect(plugin).to receive(:load_drivers).with(['/foo/bar', '/bar/foo'])
+      plugin.register
+      plugin.stop
+    end
+
+    it "should load all drivers when using a single value" do
+      mixin_settings['jdbc_driver_library'] = '/foo/bar'
+      expect(plugin).to receive(:load_drivers).with(['/foo/bar'])
+      plugin.register
+      plugin.stop
+    end
+
     it "should stop without raising exception" do
       plugin.register
       expect { plugin.stop }.to_not raise_error
@@ -199,6 +222,77 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "when fetching time data with jdbc_default_timezone set" do
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true",
+        "jdbc_default_timezone" => "America/Chicago"
+      }
+    end
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from test_table",
+      }
+    end
+
+    let(:num_rows) { 10 }
+
+    before do
+      num_rows.times do
+        db[:test_table].insert(:num => 1, :custom_time => "2015-01-01 12:00:00", :created_at => Time.now.utc)
+      end
+
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should convert the time to reflect the timezone " do
+      plugin.run(queue)
+      event = queue.pop
+      # This reflects a 6 hour time difference between UTC and America/Chicago
+      expect(event["custom_time"].time).to eq(Time.iso8601("2015-01-01T18:00:00Z"))
+    end
+  end
+
+  context "when fetching time data without jdbc_default_timezone set" do
+    let(:mixin_settings) do
+      { "jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver",
+        "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"
+      }
+    end
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from test_table",
+      }
+    end
+
+    let(:num_rows) { 1 }
+
+    before do
+      num_rows.times do
+        db.run "INSERT INTO test_table (created_at, num, custom_time) VALUES (TIMESTAMP('2015-01-01 12:00:00'), 1, TIMESTAMP('2015-01-01 12:00:00'))"
+      end
+
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should not convert the time to reflect the timezone " do
+      plugin.run(queue)
+      event = queue.pop
+      # With no timezone set, no change should occur
+      expect(event["custom_time"].time).to eq(Time.iso8601("2015-01-01T12:00:00Z"))
+    end
+  end
+
   context "when iteratively running plugin#run" do
     let(:settings) do
       {"statement" => "SELECT num, created_at FROM test_table WHERE created_at > :sql_last_start"}
@@ -379,4 +473,37 @@ describe LogStash::Inputs::Jdbc do
       expect { plugin.register }.to raise_error(Sequel::PoolTimeout)
     end
   end
+
+  context "when using logging" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from test_table", "sql_log_level" => "debug"
+      }
+    end
+
+    let(:num_rows) { 5 }
+
+    before do
+      plugin.instance_variable_set("@logger", logger)
+      allow(logger).to receive(:debug?)
+      num_rows.times do
+        db[:test_table].insert(:num => 1)
+      end
+
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    let(:logger) { double("logger") }
+
+    it "should report the staments to logging" do
+      expect(logger).to receive(:debug).with(kind_of(String)).once
+      plugin.run(queue)
+    end
+  end
+
 end

metadata

@@ -1,148 +1,176 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2015-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0.beta2
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0.beta2
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: sequel
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: sequel
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: tzinfo
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: rufus-scheduler
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: tzinfo-data
   prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rufus-scheduler
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: timecop
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: timecop
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: jdbc-derby
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: jdbc-derby
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: docker-api
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: docker-api
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/logstash/inputs/jdbc.rb
-- lib/logstash/plugin_mixins/jdbc.rb
-- spec/inputs/jdbc_spec.rb
-- logstash-input-jdbc.gemspec
-- README.md
 - CHANGELOG.md
 - Gemfile
 - LICENSE
 - NOTICE.TXT
+- README.md
+- lib/logstash/inputs/jdbc.rb
+- lib/logstash/plugin_mixins/jdbc.rb
+- logstash-input-jdbc.gemspec
+- spec/inputs/jdbc_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -155,17 +183,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This example input streams a string at a definable interval.