logstash-input-jdbc 3.0.2 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 210deedb95591e0f409fc3461af2d79bb8a384e8
-  data.tar.gz: 8543b8a26df1cd6a80fcace0f7c7e54be175cf2e
+  metadata.gz: 6212e80eff0e3db1f5f86d14b2f3222f59a35dd6
+  data.tar.gz: f23623cebfda1902907cb4f9fb5a460a73e362bc
 SHA512:
-  metadata.gz: df9c7fa5e10be322919f607a815c9fe899f05da6bb348c599e2d35390f5a10b1647ea0fe8ef60c8c05cfa524b7db3e39ae89fa256f832b5972f467c17a72c462
-  data.tar.gz: d42eccdbab0331fc8e7e834e45fb0944e22eb9e1eb583782955bf34aa7cad8aab33fe9c6c9b29f43257992f11e4e5947cd177f83d4e10cc9fb6071d80880725a
+  metadata.gz: 4f19afa4d410887fc49f5ec8c5ac30e7ad7f345c79c7b17c1f3aed69654814d259d1e00205b668baa93ab30a4c88cf595fd3b9b84f44f8bcb133ecbca234b201
+  data.tar.gz: cbda68232c8f6a7f5eddbfee91bc94d5a4f86f1a317912d85413de16f69e63e486bfa5c35ca1a804623c3b45e7021162fcea9f7fb76fae232a6de4505161a476

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 3.1.0
+  - Add an option to select the encoding data should be transform from, this will make sure all strings read from the jdbc connector are noremalized to be UTF-8 so no causing issues with later filters in LS.
+# 3.0.3
+  - Added feature to read password from external file (#120).
 # 3.0.2
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 3.0.1

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -95,6 +95,7 @@ Reading data from MySQL:
 	    jdbc_connection_string => "jdbc:mysql://host:port/database"
 	    jdbc_user => "user"
 	    jdbc_password => "password"
+      # or jdbc_password_filepath => "/path/to/my/password_file"
 	    statement => "SELECT ..."
 	    jdbc_paging_enabled => "true"
 	    jdbc_page_size => "50000"

data/lib/logstash/inputs/jdbc.rb

@@ -160,6 +160,26 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   # Whether to force the lowercasing of identifier fields
   config :lowercase_column_names, :validate => :boolean, :default => true
 
+  # The character encoding of all columns, leave empty if the columns are already properly UTF-8 
+  # encoded. Specific columns charsets using :columns_charset can override this setting.
+  config :charset, :validate => :string
+
+  # The character encoding for specific columns. This option will override the `:charset` option 
+  # for the specified columns.
+  #
+  # Example:
+  # [source,ruby]
+  # ----------------------------------
+  # input {
+  #   jdbc {
+  #     ...
+  #     columns_charset => { "column0" => "ISO-8859-1" }
+  #     ...
+  #   }
+  # }
+  # this will only convert column0 that has ISO-8859-1 as an original encoding.
+  config :columns_charset, :validate => :hash, :default => {}
+
   public
 
   def register
@@ -173,6 +193,8 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
       end
     end
 
+    @enable_encoding = !@charset.nil? || !@columns_charset.empty?
+
     # load sql_last_value from file if exists
     if @clean_run && File.exist?(@last_run_metadata_path)
       File.delete(@last_run_metadata_path)
@@ -185,6 +207,20 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     end
 
     @statement = File.read(@statement_filepath) if @statement_filepath
+
+    if (@jdbc_password_filepath and @jdbc_password)
+      raise(LogStash::ConfigurationError, "Only one of :jdbc_password, :jdbc_password_filepath may be set at a time.")
+    end
+
+    @jdbc_password = File.read(@jdbc_password_filepath).strip if @jdbc_password_filepath
+
+    if enable_encoding?
+      @converters = {}
+      @columns_charset.each do |column_name, encoding|
+        @converters[encoding] = LogStash::Util::Charset.new(encoding)
+      end
+      @converters[@charset] = LogStash::Util::Charset.new(@charset) if @charset
+    end
   end # def register
 
   def run(queue)
@@ -214,6 +250,10 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     # update default parameters
     @parameters['sql_last_value'] = @sql_last_value
     execute_statement(@statement, @parameters) do |row|
+      if enable_encoding?
+        ## do the necessary conversions to string elements
+        row = Hash[row.map { |k, v| [k.to_s, convert(k, v)] }]
+      end
       event = LogStash::Event.new(row)
       decorate(event)
       queue << event
@@ -226,4 +266,24 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     end
   end
 
+  private
+
+  def enable_encoding?
+    @enable_encoding
+  end
+
+  # make sure the encoding is uniform over fields
+  def convert(column_name, value)
+    return value unless value.is_a?(String)
+    column_charset = @columns_charset[column_name]
+    if column_charset
+      converter = @converters[column_charset]
+      converter.convert(value)
+    elsif @charset
+      converter = @converters[@charset]
+      converter.convert(value)
+    else
+      value
+    end
+  end
 end # class LogStash::Inputs::Jdbc

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -40,6 +40,9 @@ module LogStash::PluginMixins::Jdbc
     # JDBC password
     config :jdbc_password, :validate => :password
 
+    # JDBC password filename
+    config :jdbc_password_filepath, :validate => :path
+
     # JDBC enable paging
     #
     # This will cause a sql statement to be broken up into multiple queries.
@@ -260,7 +263,6 @@ module LogStash::PluginMixins::Jdbc
 
   private
   def decorate_value(value)
-
     if value.is_a?(Time)
       # transform it to LogStash::Timestamp as required by LS
       LogStash::Timestamp.new(value)
@@ -269,7 +271,7 @@ module LogStash::PluginMixins::Jdbc
       # This is slower, so we put it in as a conditional case.
       LogStash::Timestamp.new(Time.parse(value.to_s))
     else
-      value  # no-op
+      value
     end
   end
 end

data/logstash-input-jdbc.gemspec

@@ -1,9 +1,9 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '3.0.2'
+  s.version         = '3.1.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example input streams a string at a definable interval."
-  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors = ["Elastic"]
   s.email = 'info@elastic.co'
   s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"

data/spec/inputs/jdbc_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/jdbc"
 require "jdbc/derby"
@@ -29,6 +30,7 @@ describe LogStash::Inputs::Jdbc do
     db.create_table :test_table do
       DateTime :created_at
       Integer  :num
+      String   :string
       DateTime :custom_time
     end
   end
@@ -81,6 +83,41 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "when both jdbc_password and jdbc_password_filepath arguments are passed" do
+    let(:statement) { "SELECT * from test_table" }
+    let(:jdbc_password) { "secret" }
+    let(:jdbc_password_file_path) { Stud::Temporary.pathname }
+    let(:settings) { { "jdbc_password_filepath" => jdbc_password_file_path,
+                       "jdbc_password" => jdbc_password,
+                       "statement" => statement } }
+
+    it "should fail to register" do
+      expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "when jdbc_password is passed in from a file" do
+    let(:statement) { "SELECT * from test_table" }
+    let(:jdbc_password) { "secret" }
+    let(:jdbc_password_file_path) { Stud::Temporary.pathname }
+    let(:settings) { { "jdbc_password_filepath" => jdbc_password_file_path,
+                       "statement" => statement } }
+
+    before do
+      File.write(jdbc_password_file_path, jdbc_password)
+      plugin.register
+    end
+
+    after do
+      plugin.stop
+    end
+
+    it "should read in jdbc_password from file" do
+      expect(plugin.jdbc_password).to eq(jdbc_password)
+    end
+  end
+
+
   context "when neither statement and statement_filepath arguments are passed" do
     it "should fail to register" do
       expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
@@ -837,4 +874,109 @@ describe LogStash::Inputs::Jdbc do
       plugin.stop
     end
   end
+
+  context "when encoding of some columns need to be changed" do
+
+    let(:settings) {{ "statement" => "SELECT * from test_table" }}
+    let(:events)   { [] }
+    let(:row) do
+      {
+        "column0" => "foo",
+        "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
+        "column2" => 3
+      }
+    end
+
+    before(:each) do
+      allow_any_instance_of(Sequel::JDBC::Derby::Dataset).to receive(:each).and_yield(row)
+      plugin.register
+    end
+
+    after(:each) do
+      plugin.stop
+    end
+
+    it "should not convert any column by default" do
+      encoded_row = {
+        "column0" => "foo",
+        "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
+        "column2" => 3
+      }
+      expect(LogStash::Event).to receive(:new) do |row|
+        row.each do |k, v|
+          next unless v.is_a?(String)
+          expect(row[k].encoding).to eq(encoded_row[k].encoding)
+        end
+      end
+      plugin.run(events)
+    end
+
+    context "when all string columns should be encoded" do
+
+      let(:settings) do
+        {
+          "statement" => "SELECT * from test_table",
+          "charset" => "ISO-8859-1"
+        }
+      end
+
+      let(:row) do
+        {
+          "column0" => "foo".force_encoding(Encoding::ISO_8859_1),
+          "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
+          "column2" => 3
+        }
+      end
+
+      it "should transform all column string to UTF-8, default encoding" do
+        encoded_row = {
+          "column0" => "foo",
+          "column1" => "bar",
+          "column2" => 3
+        }
+        expect(LogStash::Event).to receive(:new) do |row|
+          row.each do |k, v|
+            next unless v.is_a?(String)
+            expect(row[k].encoding).to eq(encoded_row[k].encoding)
+          end
+        end
+        plugin.run(events)
+      end
+    end
+
+    context "when only an specific column should be converted" do
+
+      let(:settings) do
+        {
+          "statement" => "SELECT * from test_table",
+          "columns_charset" => { "column1" => "ISO-8859-1" }
+        }
+      end
+
+      let(:row) do
+        {
+          "column0" => "foo",
+          "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
+          "column2" => 3,
+          "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
+        }
+      end
+
+      it "should only convert the selected column" do
+        encoded_row = {
+          "column0" => "foo",
+          "column1" => "bar",
+          "column2" => 3,
+          "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
+        }
+        expect(LogStash::Event).to receive(:new) do |row|
+          row.each do |k, v|
+            next unless v.is_a?(String)
+            expect(row[k].encoding).to eq(encoded_row[k].encoding)
+          end
+        end
+        plugin.run(events)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 3.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -150,7 +150,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []