logstash-input-jdbc 4.0.2 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: afede6bb954e753352fe3138057cb32dbffc814d
-  data.tar.gz: 624feecfdfdea557f91b480acfccb7d0948a4660
+  metadata.gz: e7ea26c9ab4cd30fa89736ea255dfa9794dd5ab3
+  data.tar.gz: deaad338731a1da19e117942c27516477c1e3307
 SHA512:
-  metadata.gz: da164fbd6abf2627372925ff198ba99fd822588392365498f3543a928e1ffcdd297b3098ba30ac4d8426b06059f4c4fde8cbef4ef1f21652a9345298e27c641c
-  data.tar.gz: 4b6a2ede22c3392d6553cb2b04a7b1c26c5333722ffc7dc16ebe0fa19eee49dce7bdee9a773a4ce35aff55692942d7db06950ea571d72d37f94e165e66ea846a
+  metadata.gz: c29cf8b2fbf3e2120f6cfd94b28665fb4282443fc468a6a148373a642de6a4ccac9fc141b8ebef84e3cd2d396b11b1013ff025ccc09982e33d1bf89963ac4f50
+  data.tar.gz: 28f23e618871d17d77125a918f9787ad92a3cc4f05094b57fcf93e0cf082ba34f443a641b99939fea61689206bfcf6775e3265ae2bc49335789b4474d53b3f99

data/CHANGELOG.md

@@ -1,9 +1,13 @@
-## 4.0.2
-  - Added feature to read password from external file (#120).
+## 4.1.0
+  - Add an option to select the encoding data should be transform from,
+    this will make sure all strings read from the jdbc connector are
+    noremalized to be UTF-8 so no causing issues with later filters in LS.
 ## 4.0.1
   - Republish all the gems under jruby.
 ## 4.0.0
   - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+# 3.0.3
+  - Added feature to read password from external file (#120)
 # 3.0.2
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 3.0.1

data/lib/logstash/inputs/jdbc.rb

@@ -160,6 +160,26 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   # Whether to force the lowercasing of identifier fields
   config :lowercase_column_names, :validate => :boolean, :default => true
 
+  # The character encoding of all columns, leave empty if the columns are already properly UTF-8 
+  # encoded. Specific columns charsets using :columns_charset can override this setting.
+  config :charset, :validate => :string
+
+  # The character encoding for specific columns. This option will override the `:charset` option 
+  # for the specified columns.
+  #
+  # Example:
+  # [source,ruby]
+  # ----------------------------------
+  # input {
+  #   jdbc {
+  #     ...
+  #     columns_charset => { "column0" => "ISO-8859-1" }
+  #     ...
+  #   }
+  # }
+  # this will only convert column0 that has ISO-8859-1 as an original encoding.
+  config :columns_charset, :validate => :hash, :default => {}
+
   public
 
   def register
@@ -173,6 +193,8 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
       end
     end
 
+    @enable_encoding = !@charset.nil? || !@columns_charset.empty?
+
     # load sql_last_value from file if exists
     if @clean_run && File.exist?(@last_run_metadata_path)
       File.delete(@last_run_metadata_path)
@@ -191,6 +213,14 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     end
 
     @jdbc_password = File.read(@jdbc_password_filepath).strip if @jdbc_password_filepath
+
+    if enable_encoding?
+      @converters = {}
+      @columns_charset.each do |column_name, encoding|
+        @converters[encoding] = LogStash::Util::Charset.new(encoding)
+      end
+      @converters[@charset] = LogStash::Util::Charset.new(@charset) if @charset
+    end
   end # def register
 
   def run(queue)
@@ -220,6 +250,10 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     # update default parameters
     @parameters['sql_last_value'] = @sql_last_value
     execute_statement(@statement, @parameters) do |row|
+      if enable_encoding?
+        ## do the necessary conversions to string elements
+        row = Hash[row.map { |k, v| [k.to_s, convert(k, v)] }]
+      end
       event = LogStash::Event.new(row)
       decorate(event)
       queue << event
@@ -232,4 +266,24 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
     end
   end
 
+  private
+
+  def enable_encoding?
+    @enable_encoding
+  end
+
+  # make sure the encoding is uniform over fields
+  def convert(column_name, value)
+    return value unless value.is_a?(String)
+    column_charset = @columns_charset[column_name]
+    if column_charset
+      converter = @converters[column_charset]
+      converter.convert(value)
+    elsif @charset
+      converter = @converters[@charset]
+      converter.convert(value)
+    else
+      value
+    end
+  end
 end # class LogStash::Inputs::Jdbc

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -263,7 +263,6 @@ module LogStash::PluginMixins::Jdbc
 
   private
   def decorate_value(value)
-
     if value.is_a?(Time)
       # transform it to LogStash::Timestamp as required by LS
       LogStash::Timestamp.new(value)
@@ -272,7 +271,7 @@ module LogStash::PluginMixins::Jdbc
       # This is slower, so we put it in as a conditional case.
       LogStash::Timestamp.new(Time.parse(value.to_s))
     else
-      value  # no-op
+      value
     end
   end
 end

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '4.0.2'
+  s.version         = '4.1.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example input streams a string at a definable interval."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/jdbc"
 require "jdbc/derby"
@@ -29,6 +30,7 @@ describe LogStash::Inputs::Jdbc do
     db.create_table :test_table do
       DateTime :created_at
       Integer  :num
+      String   :string
       DateTime :custom_time
     end
   end
@@ -301,29 +303,51 @@ describe LogStash::Inputs::Jdbc do
 
     let(:settings) do
       {
-        "statement" => "SELECT * from test_table",
+        "statement" => "SELECT * from test_table WHERE custom_time > :sql_last_value",
+        "use_column_value" => true,
+        "tracking_column" => "custom_time",
+        "last_run_metadata_path" => Stud::Temporary.pathname
       }
     end
 
-    let(:num_rows) { 10 }
+    let(:hour_range) { 10..20 }
 
-    before do
-      num_rows.times do
-        db[:test_table].insert(:num => 1, :custom_time => "2015-01-01 12:00:00", :created_at => Time.now.utc)
+    it "should convert the time to reflect the timezone " do
+      last_run_value = Time.iso8601("2000-01-01T00:00:00.000Z")
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_value))
+
+      hour_range.each do |i|
+        db[:test_table].insert(:num => i, :custom_time => "2015-01-01 #{i}:00:00", :created_at => Time.now.utc)
       end
 
       plugin.register
-    end
 
-    after do
+      plugin.run(queue)
+      expected = ["2015-01-01T16:00:00.000Z",
+                  "2015-01-01T17:00:00.000Z",
+                  "2015-01-01T18:00:00.000Z",
+                  "2015-01-01T19:00:00.000Z",
+                  "2015-01-01T20:00:00.000Z",
+                  "2015-01-01T21:00:00.000Z",
+                  "2015-01-01T22:00:00.000Z",
+                  "2015-01-01T23:00:00.000Z",
+                  "2015-01-02T00:00:00.000Z",
+                  "2015-01-02T01:00:00.000Z",
+                  "2015-01-02T02:00:00.000Z"].map { |i| Time.iso8601(i) }
+      actual = queue.size.times.map { queue.pop.get("custom_time").time }
+      expect(actual).to eq(expected)
       plugin.stop
-    end
 
-    it "should convert the time to reflect the timezone " do
       plugin.run(queue)
+      expect(queue.size).to eq(0)
+      db[:test_table].insert(:num => 11, :custom_time => "2015-01-01 11:00:00", :created_at => Time.now.utc)
+      db[:test_table].insert(:num => 12, :custom_time => "2015-01-01 21:00:00", :created_at => Time.now.utc)
+      plugin.run(queue)
+      expect(queue.size).to eq(1)
       event = queue.pop
-      # This reflects a 6 hour time difference between UTC and America/Chicago
-      expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-01T18:00:00Z"))
+      expect(event.get("num")).to eq(12)
+      expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:00:00.000Z"))
+      p settings
     end
   end
 
@@ -872,4 +896,109 @@ describe LogStash::Inputs::Jdbc do
       plugin.stop
     end
   end
+
+  context "when encoding of some columns need to be changed" do
+
+    let(:settings) {{ "statement" => "SELECT * from test_table" }}
+    let(:events)   { [] }
+    let(:row) do
+      {
+        "column0" => "foo",
+        "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
+        "column2" => 3
+      }
+    end
+
+    before(:each) do
+      allow_any_instance_of(Sequel::JDBC::Derby::Dataset).to receive(:each).and_yield(row)
+      plugin.register
+    end
+
+    after(:each) do
+      plugin.stop
+    end
+
+    it "should not convert any column by default" do
+      encoded_row = {
+        "column0" => "foo",
+        "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
+        "column2" => 3
+      }
+      expect(LogStash::Event).to receive(:new) do |row|
+        row.each do |k, v|
+          next unless v.is_a?(String)
+          expect(row[k].encoding).to eq(encoded_row[k].encoding)
+        end
+      end
+      plugin.run(events)
+    end
+
+    context "when all string columns should be encoded" do
+
+      let(:settings) do
+        {
+          "statement" => "SELECT * from test_table",
+          "charset" => "ISO-8859-1"
+        }
+      end
+
+      let(:row) do
+        {
+          "column0" => "foo".force_encoding(Encoding::ISO_8859_1),
+          "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
+          "column2" => 3
+        }
+      end
+
+      it "should transform all column string to UTF-8, default encoding" do
+        encoded_row = {
+          "column0" => "foo",
+          "column1" => "bar",
+          "column2" => 3
+        }
+        expect(LogStash::Event).to receive(:new) do |row|
+          row.each do |k, v|
+            next unless v.is_a?(String)
+            expect(row[k].encoding).to eq(encoded_row[k].encoding)
+          end
+        end
+        plugin.run(events)
+      end
+    end
+
+    context "when only an specific column should be converted" do
+
+      let(:settings) do
+        {
+          "statement" => "SELECT * from test_table",
+          "columns_charset" => { "column1" => "ISO-8859-1" }
+        }
+      end
+
+      let(:row) do
+        {
+          "column0" => "foo",
+          "column1" => "bar".force_encoding(Encoding::ISO_8859_1),
+          "column2" => 3,
+          "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
+        }
+      end
+
+      it "should only convert the selected column" do
+        encoded_row = {
+          "column0" => "foo",
+          "column1" => "bar",
+          "column2" => 3,
+          "column3" => "berlin".force_encoding(Encoding::ASCII_8BIT)
+        }
+        expect(LogStash::Event).to receive(:new) do |row|
+          row.each do |k, v|
+            next unless v.is_a?(String)
+            expect(row[k].encoding).to eq(encoded_row[k].encoding)
+          end
+        end
+        plugin.run(events)
+      end
+    end
+  end
 end

metadata

@@ -1,155 +1,155 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 4.0.2
+  version: 4.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-18 00:00:00.000000000 Z
+date: 2016-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: sequel
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: sequel
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: tzinfo
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: tzinfo
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: tzinfo-data
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: tzinfo-data
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: rufus-scheduler
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: rufus-scheduler
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: timecop
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: timecop
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: jdbc-derby
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: jdbc-derby
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: docker-api
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: docker-api
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -177,12 +177,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []