logstash-integration-jdbc 5.4.2 → 5.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c54f1c22363c2d0637b44a2b68a98e22a1b2f07bc68927a4cc9bd6788a6fc009
-  data.tar.gz: 3ea76ee4fcdde152e43af6c0aab53be50dc022a8c31fce8ebac36669370a4875
+  metadata.gz: 8aae1525f173288cfa38a488fdf35425705463d6b04b44ff34f08be8abce5806
+  data.tar.gz: 21d44df47ae618e51f8f6f96812da9b1d0d9ece07c55d45fcd299a51c3e0108d
 SHA512:
-  metadata.gz: d31cef0d20064897337825bcf7ff491d3b64295c47b644d527974f458ac2c8a180d930b84b55fac6601a5f76faa10754b8975af681c4e0718ec2711e08a0d571
-  data.tar.gz: 6434512f2b958b976a12a7c4afe3d584066cb505c4818d8e340e363a7db4327a433785fc9ddfddc6c67eb2d719fc3f518346974760003873250895cb396d36e1
+  metadata.gz: 44396ddac2afadbd4221f24204f2d657c8ad7185a430367c9ddbfc4afd18e7ef26334aa96c0b9a48ca6d95f162a9d98f778d78f4495f2143fee028fc30a50e7e
+  data.tar.gz: 24ffc8ea5863a823ab6b631051a193295f651133d04fa9ab31e9dd76a77ad44fa25b470064710c07f10c262a8a101bc900a60f57e7a4d8a46f575930fc9cdbb9

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.4.4
+  - Fix: adaptations for JRuby 9.4 [#125](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/125)
+
+## 5.4.3
+  - Fix crash when metadata file can't be deleted after moving under path.data [#136](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/136)
+
 ## 5.4.2
   - Doc: described default_hash and tag_on_default_use interaction filter plugin [#122](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/122)
   - Added new settings `statement_retry_attempts` and `statement_retry_attempts_wait_time` for retry of failed sql statement execution [#123](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/123)

data/docs/input-jdbc.asciidoc

@@ -491,10 +491,12 @@ How often to validate a connection (in seconds)
 ===== `last_run_metadata_path`
 
   * Value type is <<string,string>>
-  * Default value is `"$HOME/.logstash_jdbc_last_run"`
+  * Default value is `"<path.data>/plugins/inputs/jdbc/logstash_jdbc_last_run"`
 
 Path to file with last run time
 
+In versions prior to 5.2.6 the metadata file was written to `$HOME/.logstash_jdbc_last_run`. If during a Logstash upgrade the file is found in "$HOME" it will be moved to the default location under "path.data". If the path is defined by the user then no automatic move is performed.
+
 [id="plugins-{type}s-{plugin}-lowercase_column_names"]
 ===== `lowercase_column_names`
 

data/lib/logstash/inputs/jdbc.rb

@@ -406,7 +406,7 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
         # the correct access rights
         ::File.delete(old_default_path.to_path)
         @logger.info("Successfully moved the #{old_default_path.to_path} into #{metadata_new_path.to_path}")
-      rescue e
+      rescue => e
         @logger.warn("Using new metadata file at #{metadata_new_path.to_path} but #{old_default_path} can't be removed.")
       end
     end

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 require "yaml" # persistence
+require "date"
+require "bigdecimal"
 
 module LogStash module PluginMixins module Jdbc
   class ValueTracking
@@ -31,6 +33,17 @@ module LogStash module PluginMixins module Jdbc
       set_initial
     end
 
+    if Psych::VERSION&.split('.')&.first.to_i >= 4
+      YAML_PERMITTED_CLASSES = [::DateTime, ::Time, ::BigDecimal].freeze
+      def self.load_yaml(source)
+        Psych::safe_load(source, permitted_classes: YAML_PERMITTED_CLASSES)
+      end
+    else
+      def self.load_yaml(source)
+        YAML::load(source)
+      end
+    end
+
     def set_initial
       # override in subclass
     end
@@ -112,7 +125,7 @@ module LogStash module PluginMixins module Jdbc
 
     def read
       return unless @exists
-      YAML.load(::File.read(@path))
+      ValueTracking.load_yaml(::File.read(@path))
     end
 
     def write(value)

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.4.2'
+  s.version         = '5.4.4'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/jdbc_streaming_spec.rb

@@ -259,7 +259,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
     end
@@ -277,7 +277,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"col_1" => 'from_database'}])
       end
     end
@@ -296,11 +296,11 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text", "param_field" => "1") do
+      sample({"message" => "some text", "param_field" => "1"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
 
-      sample("message" => "some text", "param_field" => "2") do
+      sample({"message" => "some text", "param_field" => "2"}) do
         expect(subject.get('new_field').nil?)
       end
     end
@@ -319,11 +319,11 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text", "param_field" => 1) do
+      sample({"message" => "some text", "param_field" => 1}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
 
-      sample("message" => "some text", "param_field" => "1") do
+      sample({"message" => "some text", "param_field" => "1"}) do
         expect(subject.get('new_field').nil?)
       end
     end
@@ -342,7 +342,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
     end

data/spec/inputs/jdbc_spec.rb

@@ -277,7 +277,7 @@ describe LogStash::Inputs::Jdbc do
       sleep 1
       for i in 0..1
         sleep 1
-        updated_last_run = YAML.load(File.read(settings["last_run_metadata_path"]))
+        updated_last_run = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))
         expect(updated_last_run).to be > last_run_time
         last_run_time = updated_last_run
       end
@@ -547,7 +547,7 @@ describe LogStash::Inputs::Jdbc do
         expect(actual).to eq(expected)
         plugin.stop
         raw_last_run_value = File.read(settings["last_run_metadata_path"])
-        last_run_value = YAML.load(raw_last_run_value)
+        last_run_value = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(raw_last_run_value)
         expect(last_run_value).to be_a(DateTime)
         expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 02:00:00.722000000 +00:00")
 
@@ -562,7 +562,7 @@ describe LogStash::Inputs::Jdbc do
         plugin.stop
         expect(event.get("num")).to eq(12)
         expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:00:00.811Z"))
-        last_run_value = YAML.load(File.read(settings["last_run_metadata_path"]))
+        last_run_value = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))
         expect(last_run_value).to be_a(DateTime)
         # verify that sub-seconds are recorded to the file
         expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 03:00:00.811000000 +00:00")
@@ -1169,19 +1169,31 @@ describe LogStash::Inputs::Jdbc do
     context "when a file exists" do
       before do
         # in a faked HOME folder save a valid previous last_run metadata file
+        allow(ENV).to receive(:[]).with(anything).and_call_original
         allow(ENV).to receive(:[]).with('HOME').and_return(fake_home)
-        File.open("#{ENV['HOME']}/.logstash_jdbc_last_run", 'w') do |file|
+
+        File.open("#{fake_home}/.logstash_jdbc_last_run", 'w') do |file|
           file.write("--- !ruby/object:DateTime '2022-03-08 08:10:00.486889000 Z'")
         end
       end
+      let(:old_path) { "#{fake_home}/.logstash_jdbc_last_run" }
+      let(:path_data) { LogStash::SETTINGS.get_value("path.data") }
+      let(:new_path) { "#{path_data}/plugins/inputs/jdbc/logstash_jdbc_last_run" }
 
       it "should be moved" do
         plugin.register
-
-        expect(::File.exist?("#{ENV['HOME']}/.logstash_jdbc_last_run")).to be false
-        path = LogStash::SETTINGS.get_value("path.data")
-        full_path = "#{path}/plugins/inputs/jdbc/logstash_jdbc_last_run"
-        expect(::File.exist?(full_path)).to be true
+        expect(::File).to_not exist(old_path)
+        expect(::File).to exist(new_path)
+      end
+      context "if the delete fails" do
+        before(:each) do
+          allow(File).to receive(:delete).and_raise ArgumentError
+        end
+        it "should be still be moved" do
+          plugin.register
+          expect(::File).to exist(old_path) # old still exists
+          expect(::File).to exist(new_path)
+        end
       end
     end
   end
@@ -1710,7 +1722,7 @@ describe LogStash::Inputs::Jdbc do
           plugin.run(queue)
 
           expect(queue.size).to eq(expected_queue_size)
-          expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
+          expect(LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
         end
       end
 
@@ -1735,7 +1747,7 @@ describe LogStash::Inputs::Jdbc do
           plugin.run(queue)
 
           expect(queue.size).to eq(expected_queue_size)
-          expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
+          expect(LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
         end
       end
     end

data/spec/plugin_mixins/jdbc/value_tracking_spec.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+require "logstash/plugin_mixins/jdbc/value_tracking"
+require "tempfile"
+
+module LogStash module PluginMixins module Jdbc
+  describe ValueTracking do
+    context "#load_yaml" do
+
+      context "with date string" do
+        let(:yaml_date_source) { "--- !ruby/object:DateTime '2023-06-15 09:59:30.558000000 +02:00'\n" }
+
+        it "should be loaded" do
+          parsed_date = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_date_source)
+          expect(parsed_date.class).to eq DateTime
+          expect(parsed_date.year).to eq 2023
+          expect(parsed_date.month).to eq 6
+          expect(parsed_date.day).to eq 15
+        end
+      end
+
+      context "with time string" do
+        let(:yaml_time_source) { "--- 2023-06-15 15:28:15.227874000 +02:00\n" }
+
+        it "should be loaded" do
+          parsed_time = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_time_source)
+          expect(parsed_time.class).to eq Time
+          expect(parsed_time.year).to eq 2023
+          expect(parsed_time.month).to eq 6
+          expect(parsed_time.day).to eq 15
+          expect(parsed_time.hour).to eq 15
+          expect(parsed_time.min).to eq 28
+          expect(parsed_time.sec).to eq 15
+        end
+      end
+
+      context "with date string" do
+        let(:yaml_bigdecimal_source) { "--- !ruby/object:BigDecimal '0:0.1e1'\n" }
+
+        it "should be loaded" do
+          parsed_bigdecimal = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_bigdecimal_source)
+          expect(parsed_bigdecimal.class).to eq BigDecimal
+          expect(parsed_bigdecimal.to_i).to eq 1
+        end
+      end
+    end
+
+    context "#build_last_value_tracker" do
+
+      let(:plugin) { double("fake plugin") }
+      let(:temp_file) { Tempfile.new('last_run_tracker') }
+
+      before(:each) do
+        allow(plugin).to receive(:record_last_run).and_return(true)
+        allow(plugin).to receive(:clean_run).and_return(false)
+        allow(plugin).to receive(:last_run_metadata_file_path).and_return(temp_file.path)
+      end
+
+      context "create numerical tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(true)
+          allow(plugin).to receive(:tracking_column_type).and_return("numeric")
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value(1)
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v).to eq 1
+        end
+      end
+
+      context "create date time tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(false)
+          allow(plugin).to receive(:jdbc_default_timezone).and_return(:something_not_nil)
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value("2023-06-15T15:28:15+02:00")
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v.class).to eq DateTime
+          expect(v.year).to eq 2023
+        end
+      end
+
+      context "create time tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(false)
+          allow(plugin).to receive(:jdbc_default_timezone).and_return(nil)
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value("2023-06-15T15:28:15+02:00")
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v.class).to eq Time
+          expect(v.min).to eq 28
+        end
+      end
+
+    end
+  end
+end end end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.4.2
+  version: 5.4.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-10 00:00:00.000000000 Z
+date: 2023-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -307,6 +307,7 @@ files:
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
 - spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
+- spec/plugin_mixins/jdbc/value_tracking_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
 - vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
 - vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
@@ -333,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Integration with JDBC - input and filter plugins
@@ -360,4 +361,5 @@ test_files:
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
 - spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
+- spec/plugin_mixins/jdbc/value_tracking_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb