logstash-input-file 4.1.18 → 4.2.4

data/lib/logstash/inputs/file.rb

@@ -332,8 +332,9 @@ class File < LogStash::Inputs::Base
     @completely_stopped.true?
   end
 
+  # The WatchedFile calls back here as `observer.listener_for(@path)`
+  # @param [String] path the identity
   def listener_for(path)
-    # path is the identity
     FileListener.new(path, self)
   end
 
@@ -376,12 +377,12 @@ class File < LogStash::Inputs::Base
   def handle_deletable_path(path)
     return if tail_mode?
     return if @completed_file_handlers.empty?
+    @logger.debug? && @logger.debug(__method__.to_s, :path => path)
     @completed_file_handlers.each { |handler| handler.handle(path) }
   end
 
   def log_line_received(path, line)
-    return unless @logger.debug?
-    @logger.debug("Received line", :path => path, :text => line)
+    @logger.debug? && @logger.debug("Received line", :path => path, :text => line)
   end
 
   def stop

data/lib/logstash/inputs/file_listener.rb

@@ -7,9 +7,9 @@ module LogStash module Inputs
   class FileListener
     attr_reader :input, :path, :data
     # construct with link back to the input plugin instance.
-    def initialize(path, input)
+    def initialize(path, input, data = nil)
       @path, @input = path, input
-      @data = nil
+      @data = data
     end
 
     def opened
@@ -36,7 +36,7 @@ module LogStash module Inputs
     def accept(data)
       # and push transient data filled dup listener downstream
       input.log_line_received(path, data)
-      input.codec.accept(dup_adding_state(data))
+      input.codec.accept(self.class.new(path, input, data))
     end
 
     def process_event(event)
@@ -45,17 +45,6 @@ module LogStash module Inputs
       input.post_process_this(event)
     end
 
-    def add_state(data)
-      @data = data
-      self
-    end
-
-    private
-
-    # duplicate and add state for downstream
-    def dup_adding_state(line)
-      self.class.new(path, input).add_state(line)
-    end
   end
 
   class FlushableListener < FileListener

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.18'
+  s.version         = '4.2.4'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -31,6 +31,7 @@ Gem::Specification.new do |s|
     s.add_runtime_dependency 'addressable'
   end
 
+  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
   s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']

data/spec/filewatch/reading_spec.rb

@@ -23,9 +23,12 @@ module FileWatch
     let(:start_new_files_at) { :end } # should be irrelevant for read mode
     let(:opts) do
       {
-        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at,
-        :delimiter => "\n", :discover_interval => discover_interval,
-        :ignore_older => 3600, :sincedb_path => sincedb_path
+        :stat_interval => stat_interval,
+        :start_new_files_at => start_new_files_at,
+        :delimiter => "\n",
+        :discover_interval => discover_interval,
+        :ignore_older => 3600,
+        :sincedb_path => sincedb_path
       }
     end
     let(:observer) { TestObserver.new }
@@ -88,7 +91,7 @@ module FileWatch
     context "when watching a directory with files using striped reading" do
       let(:file_path2) { ::File.join(directory, "2.log") }
       # use a chunk size that does not align with the line boundaries
-      let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1, :file_sort_by => "path")}
+      let(:opts) { super().merge(:file_chunk_size => 10, :file_chunk_count => 1, :file_sort_by => "path")}
       let(:lines) { [] }
       let(:observer) { TestObserver.new(lines) }
       let(:listener2) { observer.listener_for(file_path2) }
@@ -118,7 +121,7 @@ module FileWatch
     end
 
     context "when a non default delimiter is specified and it is not in the content" do
-      let(:opts) { super.merge(:delimiter => "\nø") }
+      let(:opts) { super().merge(:delimiter => "\nø") }
       let(:actions) do
         RSpec::Sequencing.run("create file") do
           File.open(file_path, "wb") { |file|  file.write("line1\nline2") }
@@ -147,8 +150,52 @@ module FileWatch
       end
     end
 
+    context "when watching directory with files and adding a new file" do
+      let(:file_path2) { ::File.join(directory, "2.log") }
+      let(:file_path3) { ::File.join(directory, "3.log") }
+
+      let(:opts) { super().merge(:file_sort_by => "last_modified") }
+      let(:lines) { [] }
+      let(:observer) { TestObserver.new(lines) }
+
+
+      let(:listener2) { observer.listener_for(file_path2) }
+      let(:listener3) { observer.listener_for(file_path3) }
+
+      let(:actions) do
+        RSpec::Sequencing.run("create12") do
+          File.open(file_path,  "w") { |file| file.write("string11\nstring12") }
+          File.open(file_path2, "w") { |file| file.write("string21\nstring22") }
+        end
+        .then("watch") do
+          reading.watch_this(watch_dir)
+        end
+        .then("wait12") do
+          wait(2).for { listener1.calls.last == :delete && listener2.calls.last == :delete }.to eq(true)
+        end
+        .then_after(2, "create3") do
+          File.open(file_path3, "w") { |file| file.write("string31\nstring32") }
+        end
+        .then("wait3") do
+          wait(2).for { listener3.calls.last == :delete }.to eq(true)
+        end
+        .then("quit") do
+          reading.quit
+        end
+      end
+
+      it "reads all (3) files" do
+        actions.activate_quietly
+        reading.subscribe(observer)
+        actions.assert_no_errors
+        expect(lines.last).to eq 'string32'
+        expect(lines.sort).to eq %w(string11 string12 string21 string22 string31 string32)
+        expect( reading.watch.watched_files_collection.paths ).to eq [ file_path, file_path2, file_path3 ]
+      end
+    end
+
     context "when watching a directory with files using exit_after_read" do
-      let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) }
+      let(:opts) { super().merge(:exit_after_read => true, :max_open_files => 2) }
       let(:file_path3) { ::File.join(directory, "3.log") }
       let(:file_path4) { ::File.join(directory, "4.log") }
       let(:file_path5) { ::File.join(directory, "5.log") }
@@ -159,40 +206,45 @@ module FileWatch
       let(:listener6) { observer.listener_for(file_path6) }
 
       it "the file is read" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines).to eq(["line1", "line2"])
       end
+
       it "multiple files are read" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4"])
       end
+
       it "multiple files are read even if max_open_files is smaller then number of files" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
         File.open(file_path5, "w") { |file| file.write("line5\nline6\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4", "line5", "line6"])
       end
+
       it "file as marked as reading_completed" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
       end
+
       it "sincedb works correctly" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         sincedb_record_fields = File.read(sincedb_path).split(" ")
         position_field_index = 3
         expect(sincedb_record_fields[position_field_index]).to eq("12")
       end
+
       it "does not include new files added after start" do
         File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
@@ -201,7 +253,6 @@ module FileWatch
         expect(listener3.lines).to eq(["line1", "line2"])
         expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
         expect(listener6.calls).to eq([])
-
       end
     
     end

data/spec/filewatch/rotate_spec.rb

@@ -219,7 +219,7 @@ module FileWatch
     end
 
     context "create + rename rotation: when a new logfile is renamed to a path we have seen before but not all content from the previous the file is read" do
-      let(:opts) { super.merge(
+      let(:opts) { super().merge(
           :file_chunk_size => line1.bytesize.succ,
           :file_chunk_count => 1
         ) }
@@ -296,7 +296,7 @@ module FileWatch
     end
 
     context "copy + truncate rotation: when a logfile is copied to a new path and truncated before the open file is fully read" do
-      let(:opts) { super.merge(
+      let(:opts) { super().merge(
           :file_chunk_size => line1.bytesize.succ,
           :file_chunk_count => 1
         ) }
@@ -370,7 +370,7 @@ module FileWatch
     end
 
     context "? rotation: when an active file is renamed inside the glob and the reading lags behind" do
-      let(:opts) { super.merge(
+      let(:opts) { super().merge(
           :file_chunk_size => line1.bytesize.succ,
           :file_chunk_count => 2
         ) }
@@ -409,7 +409,7 @@ module FileWatch
     end
 
     context "? rotation: when a not active file is rotated outside the glob before the file is read" do
-      let(:opts) { super.merge(
+      let(:opts) { super().merge(
           :close_older => 3600,
           :max_open_files => 1,
           :file_sort_by => "path"

data/spec/filewatch/settings_spec.rb

@@ -1,3 +1,6 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/friendly_durations'
+
 describe FileWatch::Settings do
 
   context "when create from options" do

data/spec/filewatch/sincedb_record_serializer_spec.rb

@@ -9,7 +9,9 @@ module FileWatch
     let(:io) { StringIO.new }
     let(:db) { Hash.new }
 
-    subject { SincedbRecordSerializer.new(SincedbRecordSerializer.days_to_seconds(14)) }
+    let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(14) }
+
+    subject { SincedbRecordSerializer.new(sincedb_value_expiry) }
 
     context "deserialize from IO" do
       it 'reads V1 records' do
@@ -82,8 +84,10 @@ module FileWatch
     end
 
     context "given a non default `sincedb_clean_after`" do
+
+      let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(2) }
+
       it "does not write expired db entries to an IO object" do
-        subject.update_sincedb_value_expiry_from_days(2)
         one_day_ago = Time.now.to_f - (1.0*24*3600)
         three_days_ago = one_day_ago - (2.0*24*3600)
         db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)

data/spec/filewatch/spec_helper.rb

@@ -117,17 +117,12 @@ module FileWatch
     class Listener
       attr_reader :path, :lines, :calls
 
-      def initialize(path)
+      def initialize(path, lines)
         @path = path
-        @lines = Concurrent::Array.new
+        @lines = lines || Concurrent::Array.new
         @calls = Concurrent::Array.new
       end
 
-      def add_lines(lines)
-        @lines = lines
-        self
-      end
-
       def accept(line)
         @lines << line
         @calls << :accept
@@ -161,12 +156,7 @@ module FileWatch
     attr_reader :listeners
 
     def initialize(combined_lines = nil)
-      listener_proc = if combined_lines.nil?
-        lambda{|k| Listener.new(k) }
-      else
-        lambda{|k| Listener.new(k).add_lines(combined_lines) }
-      end
-      @listeners = Concurrent::Hash.new {|hash, key| hash[key] = listener_proc.call(key) }
+      @listeners = Concurrent::Hash.new { |hash, key| hash[key] = new_listener(key, combined_lines) }
     end
 
     def listener_for(path)
@@ -174,6 +164,14 @@ module FileWatch
     end
 
     def clear
-      @listeners.clear; end
+      @listeners.clear
+    end
+
+    private
+
+    def new_listener(path, lines = nil)
+      Listener.new(path, lines)
+    end
+
   end
 end

data/spec/filewatch/tailing_spec.rb

@@ -10,15 +10,19 @@ module FileWatch
     let(:file_path)  { ::File.join(directory, "1#{suffix}.log") }
     let(:file_path2) { ::File.join(directory, "2#{suffix}.log") }
     let(:file_path3) { ::File.join(directory, "3#{suffix}.log") }
-    let(:max)   { 4095 }
+    let(:max) { 4095 }
     let(:stat_interval) { 0.1 }
     let(:discover_interval) { 4 }
     let(:start_new_files_at) { :end }
     let(:sincedb_path) { ::File.join(directory, "tailing.sdb") }
     let(:opts) do
       {
-        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_open_files => max,
-        :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path,
+        :stat_interval => stat_interval,
+        :start_new_files_at => start_new_files_at,
+        :max_open_files => max,
+        :delimiter => "\n",
+        :discover_interval => discover_interval,
+        :sincedb_path => sincedb_path,
         :file_sort_by => "path"
       }
     end
@@ -30,12 +34,11 @@ module FileWatch
 
     before do
       directory
-      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      wait(1.0).for { Dir.exist?(directory) }.to eq(true)
     end
 
     after do
       FileUtils.rm_rf(directory)
-      wait(1.0).for{Dir.exist?(directory)}.to eq(false)
     end
 
     describe "max open files (set to 1)" do
@@ -74,7 +77,7 @@ module FileWatch
 
       context "when close_older is set" do
         let(:wait_before_quit) { 0.8 }
-        let(:opts) { super.merge(:close_older => 0.1, :max_open_files => 1, :stat_interval => 0.1) }
+        let(:opts) { super().merge(:close_older => 0.1, :max_open_files => 1, :stat_interval => 0.1) }
         let(:suffix) { "B" }
         it "opens both files" do
           actions.activate_quietly
@@ -95,16 +98,16 @@ module FileWatch
       let(:actions) do
         RSpec::Sequencing
           .run("create file") do
-            File.open(file_path, "wb") { |file|  file.write("lineA\nlineB\n") }
+            File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") }
           end
           .then_after(0.1, "begin watching") do
             tailing.watch_this(watch_dir)
           end
-          .then_after(2, "add content") do
-            File.open(file_path, "ab") { |file|  file.write("line1\nline2\n") }
+          .then_after(1.0, "add content") do
+            File.open(file_path, "ab") { |file| file.write("line1\nline2\n") }
           end
           .then("wait") do
-            wait(0.75).for{listener1.lines}.to eq(["line1", "line2"])
+            wait(0.75).for { listener1.lines }.to_not be_empty
           end
           .then("quit") do
             tailing.quit
@@ -113,7 +116,6 @@ module FileWatch
 
       it "only the new content is read" do
         actions.activate_quietly
-        tailing.watch_this(watch_dir)
         tailing.subscribe(observer)
         actions.assert_no_errors
         expect(listener1.calls).to eq([:open, :accept, :accept])
@@ -132,7 +134,7 @@ module FileWatch
             File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
           end
           .then("wait") do
-            wait(0.75).for{listener1.lines.size}.to eq(2)
+            wait(0.75).for { listener1.lines }.to_not be_empty
           end
           .then("quit") do
             tailing.quit
@@ -154,7 +156,7 @@ module FileWatch
       # so when a stat is taken on the file an error is raised
       let(:suffix) { "E" }
       let(:quit_after) { 0.2 }
-      let(:stat)  { double("stat", :size => 100, :modified_at => Time.now.to_f, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) }
+      let(:stat)  { double("stat", :size => 100, :modified_at => Time.now.to_f, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) }
       let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) }
       before do
         allow(stat).to receive(:restat).and_raise(Errno::ENOENT)
@@ -276,7 +278,7 @@ module FileWatch
 
     context "when watching a directory with files and a file is renamed to match glob", :unix => true do
       let(:suffix) { "H" }
-      let(:opts) { super.merge(:close_older => 0) }
+      let(:opts) { super().merge(:close_older => 0) }
       let(:listener2) { observer.listener_for(file_path2) }
       let(:actions) do
         RSpec::Sequencing
@@ -344,7 +346,7 @@ module FileWatch
     end
 
     context "when close older expiry is enabled" do
-      let(:opts) { super.merge(:close_older => 1) }
+      let(:opts) { super().merge(:close_older => 1) }
       let(:suffix) { "J" }
       let(:actions) do
         RSpec::Sequencing.run("create file") do
@@ -368,7 +370,7 @@ module FileWatch
     end
 
     context "when close older expiry is enabled and after timeout the file is appended-to" do
-      let(:opts) { super.merge(:close_older => 0.5) }
+      let(:opts) { super().merge(:close_older => 0.5) }
       let(:suffix) { "K" }
       let(:actions) do
         RSpec::Sequencing
@@ -404,7 +406,7 @@ module FileWatch
     end
 
     context "when ignore older expiry is enabled and all files are already expired" do
-      let(:opts) { super.merge(:ignore_older => 1) }
+      let(:opts) { super().merge(:ignore_older => 1) }
       let(:suffix) { "L" }
       let(:actions) do
         RSpec::Sequencing
@@ -428,7 +430,7 @@ module FileWatch
 
     context "when a file is renamed before it gets activated", :unix => true do
       let(:max) { 1 }
-      let(:opts) { super.merge(:file_chunk_count => 8, :file_chunk_size => 6, :close_older => 0.1, :discover_interval => 6) }
+      let(:opts) { super().merge(:file_chunk_count => 8, :file_chunk_size => 6, :close_older => 0.1, :discover_interval => 6) }
       let(:suffix) { "M" }
       let(:start_new_files_at) { :beginning } # we are creating files and sincedb record before hand
       let(:actions) do
@@ -467,7 +469,7 @@ module FileWatch
     end
 
     context "when ignore_older is less than close_older and all files are not expired" do
-      let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.1) }
+      let(:opts) { super().merge(:ignore_older => 1, :close_older => 1.1) }
       let(:suffix) { "N" }
       let(:start_new_files_at) { :beginning }
       let(:actions) do
@@ -495,7 +497,7 @@ module FileWatch
     end
 
     context "when ignore_older is less than close_older and all files are expired" do
-      let(:opts) { super.merge(:ignore_older => 10, :close_older => 1) }
+      let(:opts) { super().merge(:ignore_older => 10, :close_older => 1) }
       let(:suffix) { "P" }
       let(:actions) do
         RSpec::Sequencing
@@ -520,7 +522,7 @@ module FileWatch
     end
 
     context "when ignore older and close older expiry is enabled and after timeout the file is appended-to" do
-      let(:opts) { super.merge(:ignore_older => 20, :close_older => 0.5) }
+      let(:opts) { super().merge(:ignore_older => 20, :close_older => 0.5) }
       let(:suffix) { "Q" }
       let(:actions) do
         RSpec::Sequencing
@@ -549,7 +551,7 @@ module FileWatch
     end
 
     context "when a non default delimiter is specified and it is not in the content" do
-      let(:opts) { super.merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") }
+      let(:opts) { super().merge(:ignore_older => 20, :close_older => 1, :delimiter => "\nø") }
       let(:suffix) { "R" }
       let(:actions) do
         RSpec::Sequencing