logstash-input-file 4.4.2 → 4.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d65df8a33a836b3e91bdd9c1d2edd86532dfb521091aa78324807e630e472ec
-  data.tar.gz: 4bbca22cb03e662490c278d3f0d827488d6563c58bd73b37a4ee59f1ad84c1cd
+  metadata.gz: 26577b41ecc7118c3f4ec2dba011fe730e44d4a74c0f45b19875ba331922bb24
+  data.tar.gz: e799e9a5e149662f3e1ffa4e902bb8c789d8d5eb7f1b90c1776445571b358a1b
 SHA512:
-  metadata.gz: a600671c2564ba9d81bed3078e9d99c82079dcd80a67143182102c268ac5396669b58a8dcfb2858c35bf698e77b06c12c546ec5f60569eb98472aa3c8086ad80
-  data.tar.gz: b3b283d5b09cbe72198921992834fd7edadf4620123c1c2912d3179ff2247ff4b8dc5c3c0f09484af76d83253bbf2173fe165e3401f42ce37762a26e1db82764
+  metadata.gz: b8ce5e67051bb9d04947d3b3e72475f2929cecf475a5aa1196556fb63fcd37c44e931ebdff1e47fa21f01030bf2a5b15a6301658dbc249633b613c681b942fd9
+  data.tar.gz: 44dc2403a6ca1dcc5c0660404e4419d5acd1f020d376cd3be8fc7d46533335c10ba10e6c0510eb9c4afa3685b281e5a4a8cfb201ac2882c500f42b3bd54681ef

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.4.3
+  - Fixes read mode to restart the read from reference stored in sincedb in case the file wasn't completely consumed. [#307](https://github.com/logstash-plugins/logstash-input-file/pull/307)
+
 ## 4.4.2
   - Doc: Fix attribute by removing extra character [#310](https://github.com/logstash-plugins/logstash-input-file/pull/310)
 

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -2,9 +2,19 @@
 
 module FileWatch module ReadMode module Handlers
   class ReadFile < Base
+
+    # seek file to which ever is furthest: either current bytes read or sincedb position
+    private
+    def seek_to_furthest_position(watched_file)
+      previous_pos = sincedb_collection.find(watched_file).position
+      watched_file.file_seek([watched_file.bytes_read, previous_pos].max)
+    end
+
+    public
     def handle_specifically(watched_file)
       if open_file(watched_file)
         add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
+        seek_to_furthest_position(watched_file)
         loop do
           break if quit?
           loop_control = watched_file.loop_control_adjusted_for_stat_size

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.4.2'
+  s.version         = '4.4.3'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filewatch/read_mode_handlers_read_file_spec.rb

@@ -36,5 +36,45 @@ module FileWatch
         processor.read_file(watched_file)
       end
     end
+
+    context "when restart from existing sincedb" do
+      let(:settings) do
+        Settings.from_options(
+          :sincedb_write_interval => 0,
+          :sincedb_path => File::NULL,
+          :file_chunk_size => 10
+        )
+      end
+
+      let(:processor) { double("fake processor") }
+      let(:observer) { TestObserver.new }
+      let(:watch) { double("watch") }
+
+      before(:each) {
+        allow(watch).to receive(:quit?).and_return(false)#.and_return(false).and_return(true)
+        allow(processor).to receive(:watch).and_return(watch)
+      }
+
+      it "read from where it left" do
+        listener = observer.listener_for(Pathname.new(pathname).to_path)
+        sut = ReadMode::Handlers::ReadFile.new(processor, sdb_collection, observer, settings)
+
+        # simulate a previous partial read of the file
+        sincedb_value = SincedbValue.new(0)
+        sincedb_value.set_watched_file(watched_file)
+        sdb_collection.set(watched_file.sincedb_key, sincedb_value)
+
+
+        # simulate a consumption of first line, (size + newline) bytes
+        sdb_collection.increment(watched_file.sincedb_key, File.readlines(pathname)[0].size + 2)
+
+        # exercise
+        sut.handle(watched_file)
+
+        # verify
+        expect(listener.lines.size).to eq(1)
+        expect(listener.lines[0]).to start_with("2010-03-12   23:51:21   SEA4   192.0.2.222   play   3914   OK")
+      end
+    end
   end
 end

data/spec/filewatch/spec_helper.rb

@@ -80,6 +80,8 @@ module FileWatch
       multiplier = amount / string.length
       string * multiplier
     end
+    def sysseek(offset, whence)
+    end
   end
 
   FIXTURE_DIR = File.join('spec', 'fixtures')

data/spec/helpers/spec_helper.rb

@@ -31,7 +31,13 @@ module FileInput
 
     def trace_for(symbol)
       params = @tracer.map {|k,v| k == symbol ? v : nil}.compact
-      params.empty? ? false : params
+      if params.empty?
+        false
+      else
+        # merge all params with same key
+        # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
+        params.reduce {|b1, b2| b1 and b2}
+      end
     end
 
     def clear

data/spec/inputs/file_tail_spec.rb

@@ -332,7 +332,7 @@ describe LogStash::Inputs::File do
           .then("wait accept") do
             wait(0.75).for {
               subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)
-            }.to eq([true]), "accept didn't"
+            }.to eq(true), "accept didn't"
           end
           .then("request a stop") do
             # without this the subject.run doesn't invokes the #exit_flush which is the only @codec.flush_mapped invocation
@@ -341,12 +341,11 @@ describe LogStash::Inputs::File do
           .then("wait for auto_flush") do
             wait(2).for {
               subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)
-                .reduce {|b1, b2| b1 and b2} # there could be multiple instances of same call, e.g. [[:accept, true], [:auto_flush, true], [:close, true], [:auto_flush, true]]
             }.to eq(true), "autoflush didn't"
           end
         subject.run(events)
         actions.assert_no_errors
-        expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true])
+        expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq(true)
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.4.2
+  version: 4.4.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-11 00:00:00.000000000 Z
+date: 2022-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement