logstash-input-jdbc 3.1.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6212e80eff0e3db1f5f86d14b2f3222f59a35dd6
-  data.tar.gz: f23623cebfda1902907cb4f9fb5a460a73e362bc
+  metadata.gz: c06e48de482487475b4325cf2967edca7b6a8dc4
+  data.tar.gz: 5da108421a7003c5c6c0a4ac9d1b5d3425d50c15
 SHA512:
-  metadata.gz: 4f19afa4d410887fc49f5ec8c5ac30e7ad7f345c79c7b17c1f3aed69654814d259d1e00205b668baa93ab30a4c88cf595fd3b9b84f44f8bcb133ecbca234b201
-  data.tar.gz: cbda68232c8f6a7f5eddbfee91bc94d5a4f86f1a317912d85413de16f69e63e486bfa5c35ca1a804623c3b45e7021162fcea9f7fb76fae232a6de4505161a476
+  metadata.gz: eecfc3a25c9de60e40a47153aeb45d9386a96568a839d4ed01d1290eaf1163b1af9e096f760ad01f5ade5e3184b9d2ee4135e021b270b3a431c5af6fc573892b
+  data.tar.gz: 8f23e33c6ad652733238125d8c5d7d485a91d308c5c60aa15a8bc0e66adfe4a1faba8cb00b1b6f25b2569b0fc12be02138695ed96c4264e1207e73aa7a21cb1c

data/CHANGELOG.md

@@ -1,7 +1,5 @@
-# 3.1.0
-  - Add an option to select the encoding data should be transform from, this will make sure all strings read from the jdbc connector are noremalized to be UTF-8 so no causing issues with later filters in LS.
-# 3.0.3
-  - Added feature to read password from external file (#120).
+## 4.0.0
+  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
 # 3.0.2
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 3.0.1

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
+
+# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec

data/README.md

@@ -95,7 +95,6 @@ Reading data from MySQL:
 	    jdbc_connection_string => "jdbc:mysql://host:port/database"
 	    jdbc_user => "user"
 	    jdbc_password => "password"
-      # or jdbc_password_filepath => "/path/to/my/password_file"
 	    statement => "SELECT ..."
 	    jdbc_paging_enabled => "true"
 	    jdbc_page_size => "50000"

data/lib/logstash/inputs/jdbc.rb

@@ -160,26 +160,6 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   # Whether to force the lowercasing of identifier fields
   config :lowercase_column_names, :validate => :boolean, :default => true
 
-  # The character encoding of all columns, leave empty if the columns are already properly UTF-8 
-  # encoded. Specific columns charsets using :columns_charset can override this setting.
-  config :charset, :validate => :string
-
-  # The character encoding for specific columns. This option will override the `:charset` option 
-  # for the specified columns.
-  #
-  # Example:
-  # [source,ruby]
-  # ----------------------------------
-  # input {
-  #   jdbc {
-  #     ...
-  #     columns_charset => { "column0" => "ISO-8859-1" }
-  #     ...
-  #   }
-  # }
-  # this will only convert column0 that has ISO-8859-1 as an original encoding.
-  config :columns_charset, :validate => :hash, :default => {}
-
   public
 
   def register
@@ -193,8 +173,6 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
       end
     end
 
-    @enable_encoding = !@charset.nil? || !@columns_charset.empty?
-
     # load sql_last_value from file if exists
     if @clean_run && File.exist?(@last_run_metadata_path)
       File.delete(@last_run_metadata_path)
@@ -207,20 +185,6 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     end
 
     @statement = File.read(@statement_filepath) if @statement_filepath
-
-    if (@jdbc_password_filepath and @jdbc_password)
-      raise(LogStash::ConfigurationError, "Only one of :jdbc_password, :jdbc_password_filepath may be set at a time.")
-    end
-
-    @jdbc_password = File.read(@jdbc_password_filepath).strip if @jdbc_password_filepath
-
-    if enable_encoding?
-      @converters = {}
-      @columns_charset.each do |column_name, encoding|
-        @converters[encoding] = LogStash::Util::Charset.new(encoding)
-      end
-      @converters[@charset] = LogStash::Util::Charset.new(@charset) if @charset
-    end
   end # def register
 
   def run(queue)
@@ -250,10 +214,6 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     # update default parameters
     @parameters['sql_last_value'] = @sql_last_value
     execute_statement(@statement, @parameters) do |row|
-      if enable_encoding?
-        ## do the necessary conversions to string elements
-        row = Hash[row.map { |k, v| [k.to_s, convert(k, v)] }]
-      end
       event = LogStash::Event.new(row)
       decorate(event)
       queue << event
@@ -266,24 +226,4 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     end
   end
 
-  private
-
-  def enable_encoding?
-    @enable_encoding
-  end
-
-  # make sure the encoding is uniform over fields
-  def convert(column_name, value)
-    return value unless value.is_a?(String)
-    column_charset = @columns_charset[column_name]
-    if column_charset
-      converter = @converters[column_charset]
-      converter.convert(value)
-    elsif @charset
-      converter = @converters[@charset]
-      converter.convert(value)
-    else
-      value
-    end
-  end
 end # class LogStash::Inputs::Jdbc

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -40,9 +40,6 @@ module LogStash::PluginMixins::Jdbc
     # JDBC password
     config :jdbc_password, :validate => :password
 
-    # JDBC password filename
-    config :jdbc_password_filepath, :validate => :path
-
     # JDBC enable paging
     #
     # This will cause a sql statement to be broken up into multiple queries.
@@ -263,6 +260,7 @@ module LogStash::PluginMixins::Jdbc
 
   private
   def decorate_value(value)
+
     if value.is_a?(Time)
       # transform it to LogStash::Timestamp as required by LS
       LogStash::Timestamp.new(value)
@@ -271,7 +269,7 @@ module LogStash::PluginMixins::Jdbc
       # This is slower, so we put it in as a conditional case.
       LogStash::Timestamp.new(Time.parse(value.to_s))
     else
-      value
+      value  # no-op
     end
   end
 end

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '3.1.0'
+  s.version         = '4.0.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example input streams a string at a definable interval."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'sequel'
   s.add_runtime_dependency 'tzinfo'

data/spec/inputs/jdbc_spec.rb

@@ -1,4 +1,3 @@
-# encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/jdbc"
 require "jdbc/derby"
@@ -30,7 +29,6 @@ describe LogStash::Inputs::Jdbc do
     db.create_table :test_table do
       DateTime :created_at
       Integer  :num
-      String   :string
       DateTime :custom_time
     end
   end
@@ -83,41 +81,6 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
-  context "when both jdbc_password and jdbc_password_filepath arguments are passed" do
-    let(:statement) { "SELECT * from test_table" }
-    let(:jdbc_password) { "secret" }
-    let(:jdbc_password_file_path) { Stud::Temporary.pathname }
-    let(:settings) { { "jdbc_password_filepath" => jdbc_password_file_path,
-                       "jdbc_password" => jdbc_password,
-                       "statement" => statement } }
-
-    it "should fail to register" do
-      expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
-    end
-  end
-
-  context "when jdbc_password is passed in from a file" do
-    let(:statement) { "SELECT * from test_table" }
-    let(:jdbc_password) { "secret" }
-    let(:jdbc_password_file_path) { Stud::Temporary.pathname }
-    let(:settings) { { "jdbc_password_filepath" => jdbc_password_file_path,
-                       "statement" => statement } }
-
-    before do
-      File.write(jdbc_password_file_path, jdbc_password)
-      plugin.register
-    end
-
-    after do
-      plugin.stop
-    end
-
-    it "should read in jdbc_password from file" do
-      expect(plugin.jdbc_password).to eq(jdbc_password)
-    end
-  end
-
-
   context "when neither statement and statement_filepath arguments are passed" do
     it "should fail to register" do
       expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
@@ -171,7 +134,7 @@ describe LogStash::Inputs::Jdbc do
 
     it "should retrieve params correctly from Event" do
       plugin.run(queue)
-      expect(queue.pop['num_param']).to eq(settings['parameters']['num_param'])
+      expect(queue.pop.get('num_param')).to eq(settings['parameters']['num_param'])
     end
   end
 
@@ -289,7 +252,7 @@ describe LogStash::Inputs::Jdbc do
     it "should convert it to LogStash::Timestamp " do
       plugin.run(queue)
       event = queue.pop
-      expect(event["custom_time"]).to be_a(LogStash::Timestamp)
+      expect(event.get("custom_time")).to be_a(LogStash::Timestamp)
     end
   end
 
@@ -325,7 +288,7 @@ describe LogStash::Inputs::Jdbc do
       plugin.run(queue)
       event = queue.pop
       # This reflects a 6 hour time difference between UTC and America/Chicago
-      expect(event["custom_time"].time).to eq(Time.iso8601("2015-01-01T18:00:00Z"))
+      expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T18:00:00Z"))
     end
   end
 
@@ -360,7 +323,7 @@ describe LogStash::Inputs::Jdbc do
       plugin.run(queue)
       event = queue.pop
       # With no timezone set, no change should occur
-      expect(event["custom_time"].time).to eq(Time.iso8601("2015-01-01T12:00:00Z"))
+      expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T12:00:00Z"))
     end
   end
 
@@ -393,7 +356,7 @@ describe LogStash::Inputs::Jdbc do
 
       actual_sum = 0
       until queue.empty? do
-        actual_sum += queue.pop['num']
+        actual_sum += queue.pop.get('num')
       end
 
       expect(actual_sum).to eq(nums.inject{|sum,x| sum + x })
@@ -874,109 +837,4 @@ describe LogStash::Inputs::Jdbc do
       plugin.stop
     end
   end
-
-  context "when encoding of some columns need to be changed" do
-
-    let(:settings) {{ "statement" => "SELECT * from test_table" }}
-    let(:events)   { [] }
-    let(:row) do
-      {
-        "column0" => "foo",
-        "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
-        "column2" => 3
-      }
-    end
-
-    before(:each) do
-      allow_any_instance_of(Sequel::JDBC::Derby::Dataset).to receive(:each).and_yield(row)
-      plugin.register
-    end
-
-    after(:each) do
-      plugin.stop
-    end
-
-    it "should not convert any column by default" do
-      encoded_row = {
-        "column0" => "foo",
-        "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
-        "column2" => 3
-      }
-      expect(LogStash::Event).to receive(:new) do |row|
-        row.each do |k, v|
-          next unless v.is_a?(String)
-          expect(row[k].encoding).to eq(encoded_row[k].encoding)
-        end
-      end
-      plugin.run(events)
-    end
-
-    context "when all string columns should be encoded" do
-
-      let(:settings) do
-        {
-          "statement" => "SELECT * from test_table",
-          "charset" => "ISO-8859-1"
-        }
-      end
-
-      let(:row) do
-        {
-          "column0" => "foo".force_encoding(Encoding::ISO_8859_1),
-          "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
-          "column2" => 3
-        }
-      end
-
-      it "should transform all column string to UTF-8, default encoding" do
-        encoded_row = {
-          "column0" => "foo",
-          "column1" => "bar",
-          "column2" => 3
-        }
-        expect(LogStash::Event).to receive(:new) do |row|
-          row.each do |k, v|
-            next unless v.is_a?(String)
-            expect(row[k].encoding).to eq(encoded_row[k].encoding)
-          end
-        end
-        plugin.run(events)
-      end
-    end
-
-    context "when only an specific column should be converted" do
-
-      let(:settings) do
-        {
-          "statement" => "SELECT * from test_table",
-          "columns_charset" => { "column1" => "ISO-8859-1" }
-        }
-      end
-
-      let(:row) do
-        {
-          "column0" => "foo",
-          "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
-          "column2" => 3,
-          "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
-        }
-      end
-
-      it "should only convert the selected column" do
-        encoded_row = {
-          "column0" => "foo",
-          "column1" => "bar",
-          "column2" => 3,
-          "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
-        }
-        expect(LogStash::Event).to receive(:new) do |row|
-          row.each do |k, v|
-            next unless v.is_a?(String)
-            expect(row[k].encoding).to eq(encoded_row[k].encoding)
-          end
-        end
-        plugin.run(events)
-      end
-    end
-  end
 end

metadata

@@ -1,156 +1,158 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-07 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-core-plugin-api
-  prerelease: false
+        version: '2.0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: sequel
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: tzinfo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: tzinfo
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: tzinfo-data
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: tzinfo-data
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rufus-scheduler
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: timecop
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: jdbc-derby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: jdbc-derby
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: docker-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: docker-api
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -171,7 +173,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: input
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -186,9 +188,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
 specification_version: 4
 summary: This example input streams a string at a definable interval.
 test_files: