logstash-input-file 4.4.2 → 4.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d65df8a33a836b3e91bdd9c1d2edd86532dfb521091aa78324807e630e472ec
-  data.tar.gz: 4bbca22cb03e662490c278d3f0d827488d6563c58bd73b37a4ee59f1ad84c1cd
+  metadata.gz: 318b69d75d8239cbc9287eac35038a576c48c8bd106e0841161d95320d262d81
+  data.tar.gz: 6e15365687edf5a185071708df3c56bdc341e67c76f6c0e4faabb516cf142bd2
 SHA512:
-  metadata.gz: a600671c2564ba9d81bed3078e9d99c82079dcd80a67143182102c268ac5396669b58a8dcfb2858c35bf698e77b06c12c546ec5f60569eb98472aa3c8086ad80
-  data.tar.gz: b3b283d5b09cbe72198921992834fd7edadf4620123c1c2912d3179ff2247ff4b8dc5c3c0f09484af76d83253bbf2173fe165e3401f42ce37762a26e1db82764
+  metadata.gz: 547e151e76062bc384d7e40fdfbc4cc79310d642c60aa0fbc2ff27971b820bb94979f78f3ef2515b2a1cff979bc9052c9202e7250257a1328ca4de97721b5da1
+  data.tar.gz: 4e6b547db17a8bc96d8741089d5c0d6b93e3169cc6b847a7133d3cbaf3de2b766c21c6683b71f7f8f6fe6150f2568ef160580391a8eee64a5825b6534df61162

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.4.4
+  - Fixes gzip file handling in read mode when run on JDK12+, including JDK17 that is bundled with Logstash 8.4+ [#312](https://github.com/logstash-plugins/logstash-input-file/pull/312)
+
+## 4.4.3
+  - Fixes read mode to restart the read from reference stored in sincedb in case the file wasn't completely consumed. [#307](https://github.com/logstash-plugins/logstash-input-file/pull/307)
+
 ## 4.4.2
   - Doc: Fix attribute by removing extra character [#310](https://github.com/logstash-plugins/logstash-input-file/pull/310)
 

data/docs/index.asciidoc

@@ -9,11 +9,6 @@ START - GENERATED VARIABLES, DO NOT EDIT!
 :release_date: %RELEASE_DATE%
 :changelog_url: %CHANGELOG_URL%
 :include_path: ../../../../logstash/docs/include
-
-ifeval::["{versioned_docs}"=="true"]
-:branch: %BRANCH%
-:ecs_version: %ECS_VERSION%
-endif::[]
 ///////////////////////////////////////////
 END - GENERATED VARIABLES, DO NOT EDIT!
 ///////////////////////////////////////////
@@ -538,9 +533,4 @@ Supported values: `us` `usec` `usecs`, e.g. "600 us", "800 usec", "900 usecs"
 [NOTE]
 `micro` `micros` and `microseconds` are not supported
 
-ifeval::["{versioned_docs}"=="true"]
-:branch: current
-:ecs_version: current
-endif::[]
-
 :default_codec!:

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -2,9 +2,19 @@
 
 module FileWatch module ReadMode module Handlers
   class ReadFile < Base
+
+    # seek file to which ever is furthest: either current bytes read or sincedb position
+    private
+    def seek_to_furthest_position(watched_file)
+      previous_pos = sincedb_collection.find(watched_file).position
+      watched_file.file_seek([watched_file.bytes_read, previous_pos].max)
+    end
+
+    public
     def handle_specifically(watched_file)
       if open_file(watched_file)
         add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
+        seek_to_furthest_position(watched_file)
         loop do
           break if quit?
           loop_control = watched_file.loop_control_adjusted_for_stat_size

data/lib/filewatch/read_mode/handlers/read_zip_file.rb

@@ -29,7 +29,7 @@ module FileWatch module ReadMode module Handlers
           gzip_stream = GZIPInputStream.new(file_stream)
           decoder = InputStreamReader.new(gzip_stream, "UTF-8")
           buffered = BufferedReader.new(decoder)
-          while (line = buffered.readLine(false))
+          while (line = buffered.readLine())
             watched_file.listener.accept(line)
             # can't quit, if we did then we would incorrectly write a 'completed' sincedb entry
             # what do we do about quit when we have just begun reading the zipped file (e.g. pipeline reloading)

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.4.2'
+  s.version         = '4.4.4'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filewatch/read_mode_handlers_read_file_spec.rb

@@ -36,5 +36,45 @@ module FileWatch
         processor.read_file(watched_file)
       end
     end
+
+    context "when restart from existing sincedb" do
+      let(:settings) do
+        Settings.from_options(
+          :sincedb_write_interval => 0,
+          :sincedb_path => File::NULL,
+          :file_chunk_size => 10
+        )
+      end
+
+      let(:processor) { double("fake processor") }
+      let(:observer) { TestObserver.new }
+      let(:watch) { double("watch") }
+
+      before(:each) {
+        allow(watch).to receive(:quit?).and_return(false)#.and_return(false).and_return(true)
+        allow(processor).to receive(:watch).and_return(watch)
+      }
+
+      it "read from where it left" do
+        listener = observer.listener_for(Pathname.new(pathname).to_path)
+        sut = ReadMode::Handlers::ReadFile.new(processor, sdb_collection, observer, settings)
+
+        # simulate a previous partial read of the file
+        sincedb_value = SincedbValue.new(0)
+        sincedb_value.set_watched_file(watched_file)
+        sdb_collection.set(watched_file.sincedb_key, sincedb_value)
+
+
+        # simulate a consumption of first line, (size + newline) bytes
+        sdb_collection.increment(watched_file.sincedb_key, File.readlines(pathname)[0].size + 2)
+
+        # exercise
+        sut.handle(watched_file)
+
+        # verify
+        expect(listener.lines.size).to eq(1)
+        expect(listener.lines[0]).to start_with("2010-03-12   23:51:21   SEA4   192.0.2.222   play   3914   OK")
+      end
+    end
   end
 end

data/spec/filewatch/spec_helper.rb

@@ -80,6 +80,8 @@ module FileWatch
       multiplier = amount / string.length
       string * multiplier
     end
+    def sysseek(offset, whence)
+    end
   end
 
   FIXTURE_DIR = File.join('spec', 'fixtures')

data/spec/helpers/spec_helper.rb

@@ -31,7 +31,13 @@ module FileInput
 
     def trace_for(symbol)
       params = @tracer.map {|k,v| k == symbol ? v : nil}.compact
-      params.empty? ? false : params
+      if params.empty?
+        false
+      else
+        # merge all params with same key
+        # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
+        params.reduce {|b1, b2| b1 and b2}
+      end
     end
 
     def clear

data/spec/inputs/file_tail_spec.rb

@@ -332,7 +332,7 @@ describe LogStash::Inputs::File do
           .then("wait accept") do
             wait(0.75).for {
               subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)
-            }.to eq([true]), "accept didn't"
+            }.to eq(true), "accept didn't"
           end
           .then("request a stop") do
             # without this the subject.run doesn't invokes the #exit_flush which is the only @codec.flush_mapped invocation
@@ -341,12 +341,11 @@ describe LogStash::Inputs::File do
           .then("wait for auto_flush") do
             wait(2).for {
               subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)
-                .reduce {|b1, b2| b1 and b2} # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
             }.to eq(true), "autoflush didn't"
           end
         subject.run(events)
         actions.assert_no_errors
-        expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true])
+        expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq(true)
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.4.2
+  version: 4.4.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-11 00:00:00.000000000 Z
+date: 2022-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement