logstash-integration-jdbc 5.4.3 → 5.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0427bfc9149fd1b4751f91bda69dcbc67fe50bbb59ba331142c52b0ac79b92df
-  data.tar.gz: 9b89e07e05f01b5f4226cbbb452a36e0edb116ca116d843d8f578508206086b8
+  metadata.gz: 8d90a719c8d965230d4a27d17aedabc24d0c853a3c4f192d1e447335ce69411d
+  data.tar.gz: 6b5027909850ccdf8bbe8873fdf284dde3acc4131da70367c88aad305f265012
 SHA512:
-  metadata.gz: 7bcc88da6e2d4de082a5547f0f273063913f2fe8e87578d7177eb4929604a66ec8be48818ba7b32a88a7bf978aad42f07c14f69f60349d82531f531c0cd77480
-  data.tar.gz: 76bc8bb2c010a6f6087e91be3280c1b83ae165449889cc425cb91b4e6fe0f9092b229f7ed37df5fb6ae7be3acbaa8c69541c610bbfff07c374a32be014ccdcbf
+  metadata.gz: 2225fadcdb6537e67119a502b1d78061dbb4d9cb7a45a899cb921ae6e77fe5f5059ff7742697399bb904fcb362b2a04fbd5818758312064ad3d0a9ad7368263d
+  data.tar.gz: a9d45af9511c5f415b1014784defdec0a2f862870dcf46907697d734ea8dbfecf465c2af3cd302d8ddef7b88ed97ce97f5ee0cc74a79689c460566c0b1ab7bab

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.4.5
+  - Pin sequel to < 5.72.0 due to ruby/bigdecimal#169 [#141](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/141)
+
+## 5.4.4
+  - Fix: adaptations for JRuby 9.4 [#125](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/125)
+
 ## 5.4.3
   - Fix crash when metadata file can't be deleted after moving under path.data [#136](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/136)
 

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 require "yaml" # persistence
+require "date"
+require "bigdecimal"
 
 module LogStash module PluginMixins module Jdbc
   class ValueTracking
@@ -31,6 +33,17 @@ module LogStash module PluginMixins module Jdbc
       set_initial
     end
 
+    if Psych::VERSION&.split('.')&.first.to_i >= 4
+      YAML_PERMITTED_CLASSES = [::DateTime, ::Time, ::BigDecimal].freeze
+      def self.load_yaml(source)
+        Psych::safe_load(source, permitted_classes: YAML_PERMITTED_CLASSES)
+      end
+    else
+      def self.load_yaml(source)
+        YAML::load(source)
+      end
+    end
+
     def set_initial
       # override in subclass
     end
@@ -112,7 +125,7 @@ module LogStash module PluginMixins module Jdbc
 
     def read
       return unless @exists
-      YAML.load(::File.read(@path))
+      ValueTracking.load_yaml(::File.read(@path))
     end
 
     def write(value)

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.4.3'
+  s.version         = '5.4.5'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -29,7 +29,7 @@ Gem::Specification.new do |s|
   # Restrict use of this plugin to versions of Logstash where support for integration plugins is present.
   s.add_runtime_dependency "logstash-core", ">= 6.5.0"
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'sequel'
+  s.add_runtime_dependency 'sequel', '< 5.72.0' # until https://github.com/ruby/bigdecimal/issues/169 is fixed
   s.add_runtime_dependency 'lru_redux' # lru cache with ttl
 
   s.add_runtime_dependency 'tzinfo'

data/spec/filters/jdbc_streaming_spec.rb

@@ -259,7 +259,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
     end
@@ -277,7 +277,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"col_1" => 'from_database'}])
       end
     end
@@ -296,11 +296,11 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text", "param_field" => "1") do
+      sample({"message" => "some text", "param_field" => "1"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
 
-      sample("message" => "some text", "param_field" => "2") do
+      sample({"message" => "some text", "param_field" => "2"}) do
         expect(subject.get('new_field').nil?)
       end
     end
@@ -319,11 +319,11 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text", "param_field" => 1) do
+      sample({"message" => "some text", "param_field" => 1}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
 
-      sample("message" => "some text", "param_field" => "1") do
+      sample({"message" => "some text", "param_field" => "1"}) do
         expect(subject.get('new_field').nil?)
       end
     end
@@ -342,7 +342,7 @@ module LogStash module Filters
       CONFIG
       end
 
-      sample("message" => "some text") do
+      sample({"message" => "some text"}) do
         expect(subject.get('new_field')).to eq([{"1" => 'from_database'}])
       end
     end

data/spec/inputs/jdbc_spec.rb

@@ -277,7 +277,7 @@ describe LogStash::Inputs::Jdbc do
       sleep 1
       for i in 0..1
         sleep 1
-        updated_last_run = YAML.load(File.read(settings["last_run_metadata_path"]))
+        updated_last_run = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))
         expect(updated_last_run).to be > last_run_time
         last_run_time = updated_last_run
       end
@@ -547,7 +547,7 @@ describe LogStash::Inputs::Jdbc do
         expect(actual).to eq(expected)
         plugin.stop
         raw_last_run_value = File.read(settings["last_run_metadata_path"])
-        last_run_value = YAML.load(raw_last_run_value)
+        last_run_value = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(raw_last_run_value)
         expect(last_run_value).to be_a(DateTime)
         expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 02:00:00.722000000 +00:00")
 
@@ -562,7 +562,7 @@ describe LogStash::Inputs::Jdbc do
         plugin.stop
         expect(event.get("num")).to eq(12)
         expect(event.get("custom_time").time).to eq(Time.iso8601("2015-01-02T03:00:00.811Z"))
-        last_run_value = YAML.load(File.read(settings["last_run_metadata_path"]))
+        last_run_value = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))
         expect(last_run_value).to be_a(DateTime)
         # verify that sub-seconds are recorded to the file
         expect(last_run_value.strftime("%F %T.%N %Z")).to eq("2015-01-02 03:00:00.811000000 +00:00")
@@ -1169,7 +1169,7 @@ describe LogStash::Inputs::Jdbc do
     context "when a file exists" do
       before do
         # in a faked HOME folder save a valid previous last_run metadata file
-        allow(ENV).to receive(:[]).and_call_original
+        allow(ENV).to receive(:[]).with(anything).and_call_original
         allow(ENV).to receive(:[]).with('HOME').and_return(fake_home)
 
         File.open("#{fake_home}/.logstash_jdbc_last_run", 'w') do |file|
@@ -1722,7 +1722,7 @@ describe LogStash::Inputs::Jdbc do
           plugin.run(queue)
 
           expect(queue.size).to eq(expected_queue_size)
-          expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
+          expect(LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))).to eq(expected_queue_size)
         end
       end
 
@@ -1747,7 +1747,7 @@ describe LogStash::Inputs::Jdbc do
           plugin.run(queue)
 
           expect(queue.size).to eq(expected_queue_size)
-          expect(YAML.load(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
+          expect(LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(File.read(settings["last_run_metadata_path"]))).to eq(last_run_value + expected_queue_size)
         end
       end
     end

data/spec/plugin_mixins/jdbc/value_tracking_spec.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+require "logstash/plugin_mixins/jdbc/value_tracking"
+require "tempfile"
+
+module LogStash module PluginMixins module Jdbc
+  describe ValueTracking do
+    context "#load_yaml" do
+
+      context "with date string" do
+        let(:yaml_date_source) { "--- !ruby/object:DateTime '2023-06-15 09:59:30.558000000 +02:00'\n" }
+
+        it "should be loaded" do
+          parsed_date = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_date_source)
+          expect(parsed_date.class).to eq DateTime
+          expect(parsed_date.year).to eq 2023
+          expect(parsed_date.month).to eq 6
+          expect(parsed_date.day).to eq 15
+        end
+      end
+
+      context "with time string" do
+        let(:yaml_time_source) { "--- 2023-06-15 15:28:15.227874000 +02:00\n" }
+
+        it "should be loaded" do
+          parsed_time = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_time_source)
+          expect(parsed_time.class).to eq Time
+          expect(parsed_time.year).to eq 2023
+          expect(parsed_time.month).to eq 6
+          expect(parsed_time.day).to eq 15
+          expect(parsed_time.hour).to eq 15
+          expect(parsed_time.min).to eq 28
+          expect(parsed_time.sec).to eq 15
+        end
+      end
+
+      context "with date string" do
+        let(:yaml_bigdecimal_source) { "--- !ruby/object:BigDecimal '0:0.1e1'\n" }
+
+        it "should be loaded" do
+          parsed_bigdecimal = LogStash::PluginMixins::Jdbc::ValueTracking.load_yaml(yaml_bigdecimal_source)
+          expect(parsed_bigdecimal.class).to eq BigDecimal
+          expect(parsed_bigdecimal.to_i).to eq 1
+        end
+      end
+    end
+
+    context "#build_last_value_tracker" do
+
+      let(:plugin) { double("fake plugin") }
+      let(:temp_file) { Tempfile.new('last_run_tracker') }
+
+      before(:each) do
+        allow(plugin).to receive(:record_last_run).and_return(true)
+        allow(plugin).to receive(:clean_run).and_return(false)
+        allow(plugin).to receive(:last_run_metadata_file_path).and_return(temp_file.path)
+      end
+
+      context "create numerical tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(true)
+          allow(plugin).to receive(:tracking_column_type).and_return("numeric")
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value(1)
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v).to eq 1
+        end
+      end
+
+      context "create date time tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(false)
+          allow(plugin).to receive(:jdbc_default_timezone).and_return(:something_not_nil)
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value("2023-06-15T15:28:15+02:00")
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v.class).to eq DateTime
+          expect(v.year).to eq 2023
+        end
+      end
+
+      context "create time tracker" do
+        before(:each) do
+          allow(plugin).to receive(:use_column_value).and_return(false)
+          allow(plugin).to receive(:jdbc_default_timezone).and_return(nil)
+        end
+
+        it "should write correctly" do
+          tracker = ValueTracking.build_last_value_tracker(plugin)
+          tracker.set_value("2023-06-15T15:28:15+02:00")
+          tracker.write
+
+          temp_file.rewind
+          v = ValueTracking.load_yaml(::File.read(temp_file.path))
+          expect(v.class).to eq Time
+          expect(v.min).to eq 28
+        end
+      end
+
+    end
+  end
+end end end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.4.3
+  version: 5.4.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-26 00:00:00.000000000 Z
+date: 2023-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -75,17 +75,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 5.72.0
   name: sequel
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 5.72.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -307,6 +307,7 @@ files:
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
 - spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
+- spec/plugin_mixins/jdbc/value_tracking_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb
 - vendor/jar-dependencies/org/apache/derby/derby/10.14.1.0/derby-10.14.1.0.jar
 - vendor/jar-dependencies/org/apache/derby/derbyclient/10.14.1.0/derbyclient-10.14.1.0.jar
@@ -360,4 +361,5 @@ test_files:
 - spec/inputs/integration/integ_spec.rb
 - spec/inputs/jdbc_spec.rb
 - spec/plugin_mixins/jdbc/timezone_proxy_spec.rb
+- spec/plugin_mixins/jdbc/value_tracking_spec.rb
 - spec/plugin_mixins/jdbc_streaming/parameter_handler_spec.rb