logstash-input-file 4.1.1 → 4.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4177a21fb94dc920a0226ca698ed1bf5a6d96203a2582de150f1b1984b6ab8d
-  data.tar.gz: 8cdbb99975efb2bda21ed8d1745f5f5f668a6275c89e35718e2a312365cdfd5a
+  metadata.gz: e860fa4a6695373c1ea4f70465392f3a95b290aefc499b0e2b5732dd99f5a0c6
+  data.tar.gz: 840ac241383e9a0777867da86a71ef3045c9201b0e26074526a8ace5033ee20b
 SHA512:
-  metadata.gz: 4552c42974950d4f9490a13883ab77118e21b1f54fa1b7d3cfd83e99191660f15811df0364fe4ee8948fbffa7370aacc67ddc965dc2d02b44992586392bd9b63
-  data.tar.gz: '00287236b17611e36d6754a265507aa6a31c18e76152e4265853ca7e1c36870865d298e66db905b87dd860b769f754a745abbf4f74eafb2511f91341d3a71c2a'
+  metadata.gz: fc311c3ecfe954c669d585cfb6791dfe901170b223e080540e4ecfbdec6a010df76ee3102dfbb732abab34b5cf3422c95b02452d1dddea125b7991be07f4a020
+  data.tar.gz: 97990bd44ba64927d16d70c2f7325d578d0b30798860f15a614c8aa6f73cf6ec9ec8ac311b80cabe40ae2ab5ef07372a816675ed4daa3128cdd97e49591828bd

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.1.2
+  - Fix `require winhelper` error in WINDOWS.
+    [Issue #184](https://github.com/logstash-plugins/logstash-input-file/issues/184)
+  - Fix when no delimiter is found in a chunk, the chunk is reread - no forward progress
+    is made in the file.
+    [Issue #185](https://github.com/logstash-plugins/logstash-input-file/issues/185)
+
 ## 4.1.1
   - Fix JAR_VERSION read problem, prevented Logstash from starting.
     [Issue #180](https://github.com/logstash-plugins/logstash-input-file/issues/180)

data/lib/filewatch/bootstrap.rb

@@ -37,7 +37,7 @@ module FileWatch
   require "jruby_file_watch"
 
   if LogStash::Environment.windows?
-    require "winhelper"
+    require_relative "winhelper"
     FileOpener = FileExt
     InodeMixin = WindowsInode
   else
@@ -53,6 +53,8 @@ module FileWatch
     end
   end
 
+  BufferExtractResult = Struct.new(:lines, :warning, :additional)
+
   class NoSinceDBPathGiven < StandardError; end
 
   # how often (in seconds) we logger.warn a failed file open, per path.

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -5,28 +5,28 @@ module FileWatch module ReadMode module Handlers
     def handle_specifically(watched_file)
       if open_file(watched_file)
         add_or_update_sincedb_collection(watched_file) unless sincedb_collection.member?(watched_file.sincedb_key)
-        # if the `file_chunk_count` * `file_chunk_size` is less than the file size
-        # then this method will be executed multiple times
-        # and the seek is moved to just after a line boundary as recorded in the sincedb
-        # for each run - so we reset the buffer
-        watched_file.reset_buffer
-        watched_file.file_seek(watched_file.bytes_read)
         changed = false
         @settings.file_chunk_count.times do
           begin
-            lines = watched_file.buffer_extract(watched_file.file_read(@settings.file_chunk_size))
-            logger.warn("read_to_eof: no delimiter found in current chunk") if lines.empty?
+            data = watched_file.file_read(@settings.file_chunk_size)
+            result = watched_file.buffer_extract(data) # expect BufferExtractResult
+            logger.info(result.warning, result.additional) unless result.warning.empty?
             changed = true
-            lines.each do |line|
+            result.lines.each do |line|
               watched_file.listener.accept(line)
+              # sincedb position is independent from the watched_file bytes_read
               sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
             end
+            # instead of tracking the bytes_read line by line we need to track by the data read size.
+            # because we initially seek to the bytes_read not the sincedb position
+            watched_file.increment_bytes_read(data.bytesize)
           rescue EOFError
             # flush the buffer now in case there is no final delimiter
             line = watched_file.buffer.flush
             watched_file.listener.accept(line) unless line.empty?
             watched_file.listener.eof
             watched_file.file_close
+            # unset_watched_file will set sincedb_value.position to be watched_file.bytes_read
             sincedb_collection.unset_watched_file(watched_file)
             watched_file.listener.deleted
             watched_file.unwatch

data/lib/filewatch/sincedb_value.rb

@@ -2,8 +2,14 @@
 
 module FileWatch
   # Tracks the position and expiry of the offset of a file-of-interest
+  # NOTE: the `watched_file.bytes_read` and this `sincedb_value.position` can diverge
+  # At any given moment IF the `watched_file.bytes_read` is greater than `sincedb_value.position`
+  # then it is larger to account for bytes held in the `watched_file.buffer`
+  # in Tail mode if we quit the buffer is not flushed and we restart from
+  # the `sincedb_value.position` (end of the last line read).
+  # in Read mode the buffer is flushed as a line and both values should be the same.
   class SincedbValue
-    attr_reader :last_changed_at, :watched_file, :path_in_sincedb
+    attr_reader :last_changed_at, :watched_file, :path_in_sincedb, :position
 
     def initialize(position, last_changed_at = nil, watched_file = nil)
       @position = position # this is the value read from disk
@@ -21,27 +27,19 @@ module FileWatch
       @last_changed_at + duration
     end
 
-    def position
-      # either the value from disk or the current wf position
-      @watched_file.nil? ? @position : @watched_file.bytes_read
-    end
-
     def update_position(pos)
+      # called when we reset the position to bof or eof on shrink or file read complete
       touch
-      if @watched_file.nil?
-        @position = pos
-      else
-        @watched_file.update_bytes_read(pos)
-      end
+      @position = pos
+      @watched_file.update_bytes_read(pos) unless @watched_file.nil?
     end
 
     def increment_position(pos)
+      # called when actual lines are sent to the observer listener
+      # this gets serialized as its a more true indication of position than
+      # chunk read size
       touch
-      if watched_file.nil?
-        @position += pos
-      else
-        @watched_file.increment_bytes_read(pos)
-      end
+      @position += pos
     end
 
     def set_watched_file(watched_file)
@@ -69,6 +67,7 @@ module FileWatch
     end
 
     def unset_watched_file
+      # called in read mode only because we flushed any remaining bytes as a final line.
       # cache the position
       # we don't cache the path here because we know we are done with this file.
       # either due via the `delete` handling

data/lib/filewatch/tail_mode/handlers/base.rb

@@ -42,13 +42,17 @@ module FileWatch module TailMode module Handlers
       @settings.file_chunk_count.times do
         begin
           data = watched_file.file_read(@settings.file_chunk_size)
-          lines = watched_file.buffer_extract(data)
-          logger.warn("read_to_eof: no delimiter found in current chunk") if lines.empty?
+          result = watched_file.buffer_extract(data) # expect BufferExtractResult
+          logger.info(result.warning, result.additional) unless result.warning.empty?
           changed = true
-          lines.each do |line|
+          result.lines.each do |line|
             watched_file.listener.accept(line)
+            # sincedb position is now independent from the watched_file bytes_read
             sincedb_collection.increment(watched_file.sincedb_key, line.bytesize + @settings.delimiter_byte_size)
           end
+          # instead of tracking the bytes_read line by line we need to track by the data read size.
+          # because we seek to the bytes_read not the sincedb position
+          watched_file.increment_bytes_read(data.bytesize)
         rescue EOFError
           # it only makes sense to signal EOF in "read" mode not "tail"
           break

data/lib/filewatch/watched_file.rb

@@ -102,7 +102,20 @@ module FileWatch
     end
 
     def buffer_extract(data)
-      @buffer.extract(data)
+      warning, additional = "", {}
+      lines = @buffer.extract(data)
+      if lines.empty?
+        warning.concat("buffer_extract: a delimiter can't be found in current chunk")
+        warning.concat(", maybe there are no more delimiters or the delimiter is incorrect")
+        warning.concat(" or the text before the delimiter, a 'line', is very large")
+        warning.concat(", if this message is logged often try increasing the `file_chunk_size` setting.")
+        additional["delimiter"] = @settings.delimiter
+        additional["read_position"] = @bytes_read
+        additional["bytes_read_count"] = data.bytesize
+        additional["last_known_file_size"] = @last_stat_size
+        additional["file_path"] = @path
+      end
+      BufferExtractResult.new(lines, warning, additional)
     end
 
     def increment_bytes_read(delta)

data/lib/logstash/inputs/file.rb

@@ -142,7 +142,7 @@ class File < LogStash::Inputs::Base
 
   # When the file input discovers a file that was last modified
   # before the specified timespan in seconds, the file is ignored.
-  # After it's discovery, if an ignored file is modified it is no
+  # After its discovery, if an ignored file is modified it is no
   # longer ignored and any new data is read. By default, this option is
   # disabled. Note this unit is in seconds.
   config :ignore_older, :validate => :number

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.1'
+  s.version         = '4.1.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filewatch/buftok_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require_relative 'spec_helper'
 
 describe FileWatch::BufferedTokenizer do

data/spec/filewatch/reading_spec.rb

@@ -1,4 +1,4 @@
-
+# encoding: utf-8
 require 'stud/temporary'
 require_relative 'spec_helper'
 require 'filewatch/observing_read'
@@ -95,6 +95,25 @@ module FileWatch
       end
     end
 
+    context "when a non default delimiter is specified and it is not in the content" do
+      let(:opts) { super.merge(:delimiter => "\nø") }
+
+      it "the file is opened, data is read, but no lines are found initially, at EOF the whole file becomes the line" do
+        File.open(file_path, "wb") { |file|  file.write("line1\nline2") }
+        actions.activate
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        listener = observer.listener_for(file_path)
+        expect(listener.calls).to eq([:open, :accept, :eof, :delete])
+        expect(listener.lines).to eq(["line1\nline2"])
+        sincedb_record_fields = File.read(sincedb_path).split(" ")
+        position_field_index = 3
+        # tailing, no delimiter, we are expecting one, if it grows we read from the start.
+        # there is an info log telling us that no lines were seen but we can't test for it.
+        expect(sincedb_record_fields[position_field_index]).to eq("11")
+      end
+    end
+
     describe "reading fixtures" do
       let(:directory) { FIXTURE_DIR }
 

data/spec/filewatch/spec_helper.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "rspec_sequencing"
 require 'rspec/wait'
 require "logstash/devutils/rspec/spec_helper"
@@ -118,3 +119,8 @@ module FileWatch
       @listeners.clear; end
   end
 end
+
+ENV["LOG_AT"].tap do |level|
+  LogStash::Logging::Logger::configure_logging(level) unless level.nil?
+end
+

data/spec/filewatch/tailing_spec.rb

@@ -1,11 +1,8 @@
-
+# encoding: utf-8
 require 'stud/temporary'
 require_relative 'spec_helper'
 require 'filewatch/observing_tail'
 
-LogStash::Logging::Logger::configure_logging("WARN")
-# LogStash::Logging::Logger::configure_logging("DEBUG")
-
 module FileWatch
   describe Watch do
     before(:all) do
@@ -58,7 +55,6 @@ module FileWatch
       end
 
       context "when max_active is 1" do
-
         it "without close_older set, opens only 1 file" do
           actions.activate
           tailing.watch_this(watch_dir)
@@ -434,6 +430,33 @@ module FileWatch
         expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
       end
     end
+
+    context "when a non default delimiter is specified and it is not in the content" do
+      let(:opts) { super.merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") }
+      before do
+        RSpec::Sequencing
+          .run("create file") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2") }
+          end
+          .then("start watching before file ages more than close_older") do
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(2.1, "quit after allowing time to close the file") do
+            tailing.quit
+          end
+      end
+
+      it "the file is opened, data is read, but no lines are found, the file times out" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :timed_out])
+        expect(observer.listener_for(file_path).lines).to eq([])
+        sincedb_record_fields = File.read(sincedb_path).split(" ")
+        position_field_index = 3
+        # tailing, no delimiter, we are expecting one, if it grows we read from the start.
+        # there is an info log telling us that no lines were seen but we can't test for it.
+        expect(sincedb_record_fields[position_field_index]).to eq("0")
+      end
+    end
   end
 end
 

data/spec/filewatch/watched_file_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'stud/temporary'
 require_relative 'spec_helper'
 

data/spec/filewatch/watched_files_collection_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require_relative 'spec_helper'
 
 module FileWatch

data/spec/filewatch/winhelper_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "stud/temporary"
 require "fileutils"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.1.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-01 00:00:00.000000000 Z
+date: 2018-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement