logstash-input-file 4.1.6 → 4.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52327c9fa0774f1a563698c9db6e9e1b5f6b0abebd7b80674ae4616c800a45b4
-  data.tar.gz: cbf40e29bb2fb480cfd96edab5a2b287a0ad6c3ea18e13a0ada2a6850ed30a25
+  metadata.gz: 452117d6e30aa2df0ee9aa1f7c663a4bed00569489a53ba479f8b68789bb9051
+  data.tar.gz: 1724d57ff775ed4df8ef3821cafdb044163a8b6c246269269db4e7ef7d1bbe00
 SHA512:
-  metadata.gz: fe35ff00a3b0063c183932d148d4eb9fe46f3e51b1db620cbdaa2db9cc479d6639830469576afde037282662cbda2ecc8c5a96f6c0bf880f4f963f191c54f9f9
-  data.tar.gz: 227fe2d426d7493611946f17b299c2602f9f26da08b90b04f3e75402e61d088e2edaf123bd3fd4d9dc9f8a2fbc3840b432bb9f8e1629a78ce8f8db9d71664a58
+  metadata.gz: 555004eb3d45f7042703e1adc61b027a009baef5578ee6d3e4d22318c0a26244c47da571b68f55041851e82b305dbfcb7fd08d4888e4f85ca95cce63627b3d51
+  data.tar.gz: 7a4988cf0671f09eb1e37ed007adbee6aeb3f097f449e6246f04ac9e40dde80139343b0ae1f5321fd94f540b15b05e2dee676d208b0c71b944e8a9512ae985e2

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 4.1.7
+  - Fixed problem in rotation handling where the target file being rotated was
+  subjected to the start_position setting when it must always start from the beginning.
+  [Issue #214](https://github.com/logstash-plugins/logstash-input-file/issues/214)
+
 ## 4.1.6
   - Fixed Errno::ENOENT exception in Discoverer. [Issue #204](https://github.com/logstash-plugins/logstash-input-file/issues/204)
 
@@ -15,7 +20,7 @@
     was possible to read into memory allocated but not filled with data resulting
     in ASCII NUL (0) bytes in the message field. Now, files are read up to the
     size as given by the remote filesystem client. Applies to tail and read modes.
-    
+
 ## 4.1.3
   - Fixed `read` mode of regular files sincedb write is requested in each read loop
     iteration rather than waiting for the end-of-file to be reached. Note: for gz files,

data/lib/filewatch/discoverer.rb

@@ -56,7 +56,7 @@ module FileWatch
     end
 
     def discover_any_files(path, ongoing)
-      fileset = Dir.glob(path).select{|f| File.file?(f) && !File.symlink?(f)}
+      fileset = Dir.glob(path).select{|f| File.file?(f)}
       logger.trace("discover_files",  "count" => fileset.size)
       fileset.each do |file|
         pathname = Pathname.new(file)

data/lib/filewatch/read_mode/processor.rb

@@ -80,6 +80,8 @@ module FileWatch module ReadMode
       end
     end
 
+    ## TODO add process_rotation_in_progress
+
     def process_active(watched_files)
       logger.trace("Active processing")
       # Handles watched_files in the active state.

data/lib/filewatch/tail_mode/processor.rb

@@ -168,14 +168,16 @@ module FileWatch module TailMode
         potential_sdb_value =  @sincedb_collection.get(potential_key)
         logger.trace(">>> Rotation In Progress", "watched_file" => watched_file.details, "found_sdb_value" => sdb_value, "potential_key" => potential_key, "potential_sdb_value" => potential_sdb_value)
         if potential_sdb_value.nil?
+          logger.trace("---------- >>>> Rotation In Progress: rotating as existing file")
+          watched_file.rotate_as_file
+          trace_message = "---------- >>>> Rotation In Progress: no potential sincedb value "
           if sdb_value.nil?
-            logger.trace("---------- >>> Rotation In Progress: rotating as initial file, no potential sincedb value AND no found sincedb value")
-            watched_file.rotate_as_initial_file
+            trace_message.concat("AND no found sincedb value")
           else
-            logger.trace("---------- >>>> Rotation In Progress: rotating as existing file, no potential sincedb value BUT found sincedb value")
-            watched_file.rotate_as_file
+            trace_message.concat("BUT found sincedb value")
             sdb_value.clear_watched_file
           end
+          logger.trace(trace_message)
           new_sdb_value = SincedbValue.new(0)
           new_sdb_value.set_watched_file(watched_file)
           @sincedb_collection.set(potential_key, new_sdb_value)

data/lib/filewatch/watched_file.rb

@@ -76,12 +76,6 @@ module FileWatch
       @sdb_key_v1 = @stat.inode_struct
     end
 
-    def rotate_as_initial_file
-      # rotation, when no sincedb record exists for new inode - we have never seen this inode before.
-      rotate_as_file
-      @initial = true
-    end
-
     def rotate_as_file(bytes_read = 0)
       # rotation, when a sincedb record exists for new inode, but no watched file to rotate from
       # probably caused by a deletion detected in the middle of the rename cascade

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.6'
+  s.version         = '4.1.7'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filewatch/rotate_spec.rb

@@ -28,7 +28,7 @@ module FileWatch
     let(:max)   { 4095 }
     let(:stat_interval) { 0.01 }
     let(:discover_interval) { 15 }
-    let(:start_new_files_at) { :beginning }
+    let(:start_new_files_at) { :end }
     let(:sincedb_path) { directory.join("tailing.sdb") }
     let(:opts) do
       {
@@ -447,5 +447,49 @@ module FileWatch
         expect(listener3.lines.size).to eq(0)
       end
     end
+
+    context "? rotation: when an active file is renamed inside the glob - issue 214" do
+      let(:watch_dir) { directory.join("*L.log") }
+      let(:file_path) { directory.join("1L.log") }
+      let(:second_file) { directory.join("2L.log") }
+      subject { described_class.new(conf) }
+      let(:listener1) { observer.listener_for(file1_path) }
+      let(:listener2) { observer.listener_for(second_file.to_path) }
+      let(:stat_interval) { 0.25 }
+      let(:discover_interval) { 1 }
+      let(:line4) { "Line 4 - Some other non lorem ipsum content" }
+      let(:actions) do
+        RSpec::Sequencing
+        .run_after(0.75, "create file") do
+          file_path.open("wb") { |file|  file.puts(line1); file.puts(line2) }
+        end
+        .then_after(0.5, "rename") do
+          file_path.rename(second_file)
+          file_path.open("wb") { |file|  file.puts("#{line3}") }
+        end
+        .then("wait for expectations to be met") do
+          wait(2.0).for{listener1.lines.size + listener2.lines.size}.to eq(3)
+        end
+        .then_after(0.5, "rename again") do
+          file_path.rename(second_file)
+          file_path.open("wb") { |file|  file.puts("#{line4}") }
+        end
+        .then("wait for expectations to be met") do
+          wait(2.0).for{listener1.lines.size + listener2.lines.size}.to eq(4)
+        end
+        .then("quit") do
+          tailing.quit
+        end
+      end
+
+      it "content is read correctly, the renamed file is not reread from scratch" do
+        actions.activate_quietly
+        tailing.watch_this(watch_dir.to_path)
+        tailing.subscribe(observer)
+        actions.assert_no_errors
+        expect(listener1.lines).to eq([line1, line2, line3, line4])
+        expect(listener2.lines).to eq([])
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.1.6
+  version: 4.1.7
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-09-10 00:00:00.000000000 Z
+date: 2018-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement