logstash-integration-jdbc 5.2.5 → 5.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b74796568bc1df12ae5015cfcf38aeb43971b3bd3eda2dcc668104decc69b6a2
-  data.tar.gz: ae3a1f82f7354798692933a2426977ea2e17f7335a74d02eb09a376e4421af6b
+  metadata.gz: c9cc41a07a8cb2f7e4b9a6993dd4d0cd69761b3fa95ba81238d0dd6267367311
+  data.tar.gz: 32866a340d87d91734586b3324a1349437abc624deb6ff9883ab998b0a4524a6
 SHA512:
-  metadata.gz: d198c08e74dc4702dae5de1ca4661d68d10298ff497e44739e7cc775d575bfc9fe825154a9de6f2497e60fb3c694f06d36d8971ecf9935e13592c959d6c13c53
-  data.tar.gz: ccd725393201ec5282644f003880fa0449a027942442698fe6e1758f25335ff585efb73d0f52716c9698eb19990ef67a15b21fef90255473b39683339520796a
+  metadata.gz: b3513ba2d9ade38a829ec0b0c47ad6c0978d0256e03fb4ac26e136a064db65b08a19b7ee8dd7dabe26851739877b82f0f9a550b2328473b8ffd62f194d569ebb
+  data.tar.gz: bf71a208f07bc00c213724cbcf175a9ede2f87bf4eaf85a926ee8e9f87b322e2dd79164db1db84ec60633a0b1cfa961e8fc6d3889c8ca25ea220fa437285af5d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.2.6
+  - Fix: change default path of 'last_run_metadata_path' to be rooted in the LS data.path folder and not in $HOME [#106](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/106)
+
 ## 5.2.5
   - Fix: do not execute more queries with debug logging [#109](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/109)
 

data/lib/logstash/filters/jdbc_static.rb

@@ -175,9 +175,23 @@ module LogStash module Filters class JdbcStatic < LogStash::Filters::Base
   private
 
   def prepare_data_dir
+    # cleanup existing Derby file left behind in $HOME
+    derby_log = "#{ENV['HOME']}/derby.log"
+    if ::File.exist?(derby_log)
+      begin
+        ::File.delete(derby_log)
+      rescue Errno::EPERM => e
+        @logger.warn("Can't delete temporary file #{derby_log} due to access permissions")
+      rescue e
+        @logger.warn("Can't delete temporary file #{derby_log}", {message => e.message})
+      end
+    end
+
     # later, when local persistent databases are allowed set this property to LS_HOME/data/jdbc-static/
     # must take multi-pipelines into account and more than one config using the same jdbc-static settings
-    java.lang.System.setProperty("derby.system.home", ENV["HOME"])
+    path_data = Pathname.new(LogStash::SETTINGS.get_value("path.data")).join("plugins", "shared", "derby_home")
+    path_data.mkpath
+    java.lang.System.setProperty("derby.system.home", path_data.to_path)
     logger.info("derby.system.home is: #{java.lang.System.getProperty("derby.system.home")}")
   end
 

data/lib/logstash/inputs/jdbc.rb

@@ -9,6 +9,7 @@ require "logstash/plugin_mixins/ecs_compatibility_support/target_check"
 require "logstash/plugin_mixins/validator_support/field_reference_validation_adapter"
 
 require "logstash/plugin_mixins/event_support/event_factory_adapter"
+require "fileutils"
 
 # this require_relative returns early unless the JRuby version is between 9.2.0.0 and 9.2.8.0
 require_relative "tzinfo_jruby_patch"
@@ -178,8 +179,10 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   # exactly once.
   config :schedule, :validate => :string
 
-  # Path to file with last run time
-  config :last_run_metadata_path, :validate => :string, :default => "#{ENV['HOME']}/.logstash_jdbc_last_run"
+  # Path to file with last run time.
+  # The default will write file to `<path.data>/plugins/inputs/jdbc/logstash_jdbc_last_run`
+  # NOTE: it must be a file path and not a directory path
+  config :last_run_metadata_path, :validate => :string
 
   # Use an incremental column value rather than a timestamp
   config :use_column_value, :validate => :boolean, :default => false
@@ -230,11 +233,33 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
   config :target, :validate => :field_reference, :required => false
 
   attr_reader :database # for test mocking/stubbing
+  attr_reader :last_run_metadata_file_path # path to the file used as last run storage
 
   public
 
   def register
     @logger = self.logger
+
+    if @record_last_run
+      if @last_run_metadata_path.nil?
+        logstash_data_path = LogStash::SETTINGS.get_value("path.data")
+        logstash_data_path = Pathname.new(logstash_data_path).join("plugins", "inputs", "jdbc")
+        # Ensure that the filepath exists before writing, since it's deeply nested.
+        logstash_data_path.mkpath
+        logstash_data_file_path = logstash_data_path.join("logstash_jdbc_last_run")
+
+        ensure_default_metadatafile_location(logstash_data_file_path)
+
+        @last_run_metadata_file_path = logstash_data_file_path.to_path
+      else
+        #  validate the path is a file and not a directory
+        if Pathname.new(@last_run_metadata_path).directory?
+          raise LogStash::ConfigurationError.new("The \"last_run_metadata_path\" argument must point to a file, received a directory: \"#{last_run_metadata_path}\"")
+        end
+        @last_run_metadata_file_path = @last_run_metadata_path
+      end
+    end
+
     require "rufus/scheduler"
     prepare_jdbc_connection
 
@@ -363,4 +388,23 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base
       value
     end
   end
+
+  def ensure_default_metadatafile_location(metadata_new_path)
+    old_default_path = Pathname.new("#{ENV['HOME']}/.logstash_jdbc_last_run")
+
+    if old_default_path.exist? && !metadata_new_path.exist?
+      # Previous versions of the plugin hosted the state file into $HOME/.logstash_jdbc_last_run.
+      # Copy in the new location
+      FileUtils.cp(old_default_path.to_path, metadata_new_path.to_path)
+      begin
+        # If there is a permission error in the delete of the old file inform the user to give
+        # the correct access rights
+        ::File.delete(old_default_path.to_path)
+        @logger.info("Successfully moved the #{old_default_path.to_path} into #{metadata_new_path.to_path}")
+      rescue e
+        @logger.warn("Using new metadata file at #{metadata_new_path.to_path} but #{old_default_path} can't be removed.")
+      end
+    end
+  end
+
 end end end # class LogStash::Inputs::Jdbc

data/lib/logstash/plugin_mixins/jdbc/value_tracking.rb

@@ -5,7 +5,7 @@ module LogStash module PluginMixins module Jdbc
   class ValueTracking
 
     def self.build_last_value_tracker(plugin)
-      handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_path) : NullFileHandler.new(plugin.last_run_metadata_path)
+      handler = plugin.record_last_run ? FileHandler.new(plugin.last_run_metadata_file_path) : NullFileHandler.new(plugin.last_run_metadata_file_path)
       if plugin.clean_run
         handler.clean
       end

data/logstash-integration-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-integration-jdbc'
-  s.version         = '5.2.5'
+  s.version         = '5.2.6'
   s.licenses = ['Apache License (2.0)']
   s.summary         = "Integration with JDBC - input and filter plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/jdbc_static_spec.rb

@@ -5,6 +5,7 @@ require "sequel"
 require "sequel/adapters/jdbc"
 require "stud/temporary"
 require "timecop"
+require "pathname"
 
 # LogStash::Logging::Logger::configure_logging("WARN")
 
@@ -85,6 +86,15 @@ module LogStash module Filters
 
     let(:ipaddr) { ".3.1.1" }
 
+    describe "verify derby path property" do
+      it "should be set into Logstash data path" do
+        plugin.register
+
+        expected = Pathname.new(LogStash::SETTINGS.get_value("path.data")).join("plugins", "shared", "derby_home").to_path
+        expect(java.lang.System.getProperty("derby.system.home")).to eq(expected)
+      end
+    end
+
     describe "non scheduled operation" do
       after { plugin.close }
 

data/spec/inputs/jdbc_spec.rb

@@ -9,6 +9,7 @@ require "timecop"
 require "stud/temporary"
 require "time"
 require "date"
+require "pathname"
 
 # We do not need to set TZ env var anymore because we can have 'Sequel.application_timezone' set to utc by default now.
 
@@ -51,6 +52,9 @@ describe LogStash::Inputs::Jdbc do
       db.drop_table(:types_table)
       db.drop_table(:test1_table)
     end
+
+    last_run_default_path = LogStash::SETTINGS.get_value("path.data")
+    FileUtils.rm_f("#{last_run_default_path}/plugins/inputs/jdbc/logstash_jdbc_last_run")
   end
 
   context "when registering and tearing down" do
@@ -1114,6 +1118,86 @@ describe LogStash::Inputs::Jdbc do
     end
   end
 
+  context "when state is persisted" do
+    context "to file" do
+      let(:settings) do
+        {
+          "statement" => "SELECT * FROM test_table",
+          "record_last_run" => true
+        }
+      end
+
+      before do
+        plugin.register
+      end
+
+      after do
+        plugin.stop
+      end
+
+      context "with default last_run_metadata_path" do
+        it "should save state in data.data subpath" do
+          path = LogStash::SETTINGS.get_value("path.data")
+          expect(plugin.last_run_metadata_file_path).to start_with(path)
+        end
+      end
+
+      context "with customized last_run_metadata_path" do
+        let(:settings) { super().merge({ "last_run_metadata_path" => Stud::Temporary.pathname })}
+
+        it "should save state in data.data subpath" do
+          expect(plugin.last_run_metadata_file_path).to start_with(settings["last_run_metadata_path"])
+        end
+      end
+    end
+
+    context "with customized last_run_metadata_path point to directory" do
+      let(:settings) do
+        path = Stud::Temporary.pathname
+        Pathname.new(path).tap {|path| path.mkpath}
+        super().merge({ "last_run_metadata_path" => path})
+      end
+
+      it "raise configuration error" do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
+  context "update the previous default last_run_metadata_path" do
+    let(:settings) do
+      {
+        "statement" => "SELECT * FROM test_table",
+        "record_last_run" => true
+      }
+    end
+
+    let(:fake_home) do
+       path = Stud::Temporary.pathname
+       Pathname.new(path).tap {|path| path.mkpath}
+       path
+    end
+
+    context "when a file exists" do
+      before do
+        # in a faked HOME folder save a valid previous last_run metadata file
+        allow(ENV).to receive(:[]).with('HOME').and_return(fake_home)
+        File.open("#{ENV['HOME']}/.logstash_jdbc_last_run", 'w') do |file|
+          file.write("--- !ruby/object:DateTime '2022-03-08 08:10:00.486889000 Z'")
+        end
+      end
+
+      it "should be moved" do
+        plugin.register
+
+        expect(::File.exist?("#{ENV['HOME']}/.logstash_jdbc_last_run")).to be false
+        path = LogStash::SETTINGS.get_value("path.data")
+        full_path = "#{path}/plugins/inputs/jdbc/logstash_jdbc_last_run"
+        expect(::File.exist?(full_path)).to be true
+      end
+    end
+  end
+
   context "when setting fetch size" do
 
     let(:settings) do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-jdbc
 version: !ruby/object:Gem::Version
-  version: 5.2.5
+  version: 5.2.6
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-16 00:00:00.000000000 Z
+date: 2022-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement