logstash-input-file 4.4.4 → 4.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 318b69d75d8239cbc9287eac35038a576c48c8bd106e0841161d95320d262d81
-  data.tar.gz: 6e15365687edf5a185071708df3c56bdc341e67c76f6c0e4faabb516cf142bd2
+  metadata.gz: eabfa7af5953763874689d9cba6691d9184de195e33dbf40425021653e8f3548
+  data.tar.gz: cdca8f6300254f1d32b327194ed5977f85bb47b6912a27170d4bbacae2d6bad8
 SHA512:
-  metadata.gz: 547e151e76062bc384d7e40fdfbc4cc79310d642c60aa0fbc2ff27971b820bb94979f78f3ef2515b2a1cff979bc9052c9202e7250257a1328ca4de97721b5da1
-  data.tar.gz: 4e6b547db17a8bc96d8741089d5c0d6b93e3169cc6b847a7133d3cbaf3de2b766c21c6683b71f7f8f6fe6150f2568ef160580391a8eee64a5825b6534df61162
+  metadata.gz: bbbe5f2f43e7ef106a67bc63ebacb71d1312574dbfac8f4566ab4fef400ad751d6be0e258d8eac2358116c6e90584ca0664b2a5d44e0d853d84ff2703f148904
+  data.tar.gz: 4a7f8f7bedc904dc059ffe8acac14da3ff63663b6a4a5e748c8750952e85a5f15dc0024d696c3de5611fd40408e8f7be7d30c8cb0c5ca9ac2f436aab55d8bb4d

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.4.6
+  - Change read mode to immediately stop consuming buffered lines when shutdown is requested [#322](https://github.com/logstash-plugins/logstash-input-file/pull/322)
+
+## 4.4.5
+  - Handle EOF when checking archive validity [#321](https://github.com/logstash-plugins/logstash-input-file/pull/321)
+
 ## 4.4.4
   - Fixes gzip file handling in read mode when run on JDK12+, including JDK17 that is bundled with Logstash 8.4+ [#312](https://github.com/logstash-plugins/logstash-input-file/pull/312)
 

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -54,6 +54,7 @@ module FileWatch module ReadMode module Handlers
             # sincedb position is independent from the watched_file bytes_read
             delta = line.bytesize + @settings.delimiter_byte_size
             sincedb_collection.increment(watched_file.sincedb_key, delta)
+            break if quit?
           end
         rescue EOFError => e
           log_error("controlled_read: eof error reading file", watched_file, e)

data/lib/filewatch/read_mode/handlers/read_zip_file.rb

@@ -71,14 +71,14 @@ module FileWatch module ReadMode module Handlers
 
     def corrupted?(watched_file)
       begin
+        start = Time.new
         file_stream = FileInputStream.new(watched_file.path)
         gzip_stream = GZIPInputStream.new(file_stream)
         buffer = Java::byte[8192].new
-        start = Time.new
         until gzip_stream.read(buffer) == -1
         end
         return false
-      rescue ZipException => e
+      rescue ZipException, Java::JavaIo::EOFException => e
         duration = Time.now - start
         logger.warn("Detected corrupted archive #{watched_file.path} file won't be processed", :message => e.message,
                     :duration => duration.round(3))

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.4.4'
+  s.version         = '4.4.6'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/helpers/spec_helper.rb

@@ -24,6 +24,12 @@ module FileInput
     f.close()
   end
 
+  def self.truncate_gzip(file_path)
+    f = File.open(file_path, "ab")
+    f.truncate(100)
+    f.close()
+  end
+
   class TracerBase
     def initialize
       @tracer = Concurrent::Array.new

data/spec/inputs/file_read_spec.rb

@@ -181,25 +181,27 @@ describe LogStash::Inputs::File do
     end
 
     context "for a compressed file" do
+      let(:tmp_directory) { Stud::Temporary.directory }
+      let(:all_files_path) { fixture_dir.join("compressed.*.*") }
+      let(:gz_file_path) { fixture_dir.join('compressed.log.gz') }
+      let(:gzip_file_path) { fixture_dir.join('compressed.log.gzip') }
+      let(:sincedb_path) { ::File.join(tmp_directory, "sincedb.db") }
+      let(:log_completed_path) { ::File.join(tmp_directory, "completed.log") }
+
       it "the file is read" do
-        file_path = fixture_dir.join('compressed.log.gz')
-        file_path2 = fixture_dir.join('compressed.log.gzip')
-        FileInput.make_fixture_current(file_path.to_path)
-        FileInput.make_fixture_current(file_path2.to_path)
-        tmpfile_path = fixture_dir.join("compressed.*.*")
-        directory = Stud::Temporary.directory
-        sincedb_path = ::File.join(directory, "readmode_C_sincedb.txt")
-        log_completed_path = ::File.join(directory, "C_completed.txt")
+        FileInput.make_fixture_current(gz_file_path.to_path)
+        FileInput.make_fixture_current(gzip_file_path.to_path)
 
         conf = <<-CONFIG
         input {
           file {
             type => "blah"
-            path => "#{tmpfile_path}"
+            path => "#{all_files_path}"
             sincedb_path => "#{sincedb_path}"
             mode => "read"
             file_completed_action => "log"
             file_completed_log_path => "#{log_completed_path}"
+            exit_after_read => true
           }
         }
         CONFIG
@@ -216,17 +218,11 @@ describe LogStash::Inputs::File do
       end
 
       it "the corrupted file is untouched" do
-        directory = Stud::Temporary.directory
-        file_path = fixture_dir.join('compressed.log.gz')
-        corrupted_file_path = ::File.join(directory, 'corrupted.gz')
-        FileUtils.cp(file_path, corrupted_file_path)
+        corrupted_file_path = ::File.join(tmp_directory, 'corrupted.gz')
+        FileUtils.cp(gz_file_path, corrupted_file_path)
 
         FileInput.corrupt_gzip(corrupted_file_path)
 
-        log_completed_path = ::File.join(directory, "C_completed.txt")
-        f = File.new(log_completed_path, "w")
-        f.close()
-
         conf = <<-CONFIG
         input {
           file {
@@ -236,11 +232,38 @@ describe LogStash::Inputs::File do
             file_completed_action => "log_and_delete"
             file_completed_log_path => "#{log_completed_path}"
             check_archive_validity => true
+            exit_after_read => true
           }
         }
         CONFIG
 
-        events = input(conf) do |pipeline, queue|
+        input(conf) do |pipeline, queue|
+          wait(1)
+          expect(IO.read(log_completed_path)).to be_empty
+        end
+      end
+
+      it "the truncated file is untouched" do
+        truncated_file_path = ::File.join(tmp_directory, 'truncated.gz')
+        FileUtils.cp(gz_file_path, truncated_file_path)
+
+        FileInput.truncate_gzip(truncated_file_path)
+
+        conf = <<-CONFIG
+        input {
+          file {
+            type => "blah"
+            path => "#{truncated_file_path}"
+            mode => "read"
+            file_completed_action => "log_and_delete"
+            file_completed_log_path => "#{log_completed_path}"
+            check_archive_validity => true
+            exit_after_read => true
+          }
+        }
+        CONFIG
+
+        input(conf) do |pipeline, queue|
           wait(1)
           expect(IO.read(log_completed_path)).to be_empty
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.4.4
+  version: 4.4.6
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-20 00:00:00.000000000 Z
+date: 2023-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -282,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Streams events from files