logstash-input-file 4.1.17 → 4.2.3

data/lib/logstash/inputs/delete_completed_file_handler.rb

@@ -2,8 +2,13 @@
 
 module LogStash module Inputs
   class DeleteCompletedFileHandler
+    def initialize(watch)
+      @watch = watch
+    end
+
     def handle(path)
       Pathname.new(path).unlink rescue nil
+      @watch.watched_files_collection.remove_paths([path])
     end
   end
 end end

data/lib/logstash/inputs/file.rb

@@ -6,6 +6,7 @@ require "logstash/codecs/identity_map_codec"
 require "pathname"
 require "socket" # for Socket.gethostname
 require "fileutils"
+require "concurrent/atomic/atomic_reference"
 
 require_relative "file/patch"
 require_relative "file_listener"
@@ -247,6 +248,9 @@ class File < LogStash::Inputs::Base
     end
   end
 
+  # @private used in specs
+  attr_reader :watcher
+
   def register
     require "addressable/uri"
     require "digest/md5"
@@ -274,8 +278,6 @@ class File < LogStash::Inputs::Base
       :check_archive_validity => @check_archive_validity,
     }
 
-    @completed_file_handlers = []
-
     @path.each do |path|
       if Pathname.new(path).relative?
         raise ArgumentError.new("File paths must be absolute, relative path specified: #{path}")
@@ -319,15 +321,10 @@ class File < LogStash::Inputs::Base
       @watcher_class = FileWatch::ObservingTail
     else
       @watcher_class = FileWatch::ObservingRead
-      if @file_completed_action.include?('log')
-        @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path)
-      end
-      if @file_completed_action.include?('delete')
-        @completed_file_handlers << DeleteCompletedFileHandler.new
-      end
     end
     @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
     @completely_stopped = Concurrent::AtomicBoolean.new
+    @queue = Concurrent::AtomicReference.new
   end # def register
 
   def completely_stopped?
@@ -335,8 +332,9 @@ class File < LogStash::Inputs::Base
     @completely_stopped.true?
   end
 
+  # The WatchedFile calls back here as `observer.listener_for(@path)`
+  # @param [String] path the identity
   def listener_for(path)
-    # path is the identity
     FileListener.new(path, self)
   end
 
@@ -344,13 +342,25 @@ class File < LogStash::Inputs::Base
     # if the pipeline restarts this input,
     # make sure previous files are closed
     stop
+
     @watcher = @watcher_class.new(@filewatch_config)
+
+    @completed_file_handlers = []
+    if read_mode?
+      if @file_completed_action.include?('log')
+        @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path)
+      end
+      if @file_completed_action.include?('delete')
+        @completed_file_handlers << DeleteCompletedFileHandler.new(@watcher.watch)
+      end
+    end
+
     @path.each { |path| @watcher.watch_this(path) }
   end
 
   def run(queue)
     start_processing
-    @queue = queue
+    @queue.set queue
     @watcher.subscribe(self) # halts here until quit is called
     # last action of the subscribe call is to write the sincedb
     exit_flush
@@ -361,18 +371,18 @@ class File < LogStash::Inputs::Base
     event.set("[@metadata][host]", @host)
     event.set("host", @host) unless event.include?("host")
     decorate(event)
-    @queue << event
+    @queue.get << event
   end
 
   def handle_deletable_path(path)
     return if tail_mode?
     return if @completed_file_handlers.empty?
+    @logger.debug? && @logger.debug(__method__.to_s, :path => path)
     @completed_file_handlers.each { |handler| handler.handle(path) }
   end
 
   def log_line_received(path, line)
-    return unless @logger.debug?
-    @logger.debug("Received line", :path => path, :text => line)
+    @logger.debug? && @logger.debug("Received line", :path => path, :text => line)
   end
 
   def stop
@@ -382,6 +392,11 @@ class File < LogStash::Inputs::Base
     end
   end
 
+  # @private used in specs
+  def queue
+    @queue.get
+  end
+
   private
 
   def build_sincedb_base_from_settings(settings)

data/lib/logstash/inputs/file_listener.rb

@@ -7,9 +7,9 @@ module LogStash module Inputs
   class FileListener
     attr_reader :input, :path, :data
     # construct with link back to the input plugin instance.
-    def initialize(path, input)
+    def initialize(path, input, data = nil)
       @path, @input = path, input
-      @data = nil
+      @data = data
     end
 
     def opened
@@ -36,7 +36,7 @@ module LogStash module Inputs
     def accept(data)
       # and push transient data filled dup listener downstream
       input.log_line_received(path, data)
-      input.codec.accept(dup_adding_state(data))
+      input.codec.accept(self.class.new(path, input, data))
     end
 
     def process_event(event)
@@ -45,17 +45,6 @@ module LogStash module Inputs
       input.post_process_this(event)
     end
 
-    def add_state(data)
-      @data = data
-      self
-    end
-
-    private
-
-    # duplicate and add state for downstream
-    def dup_adding_state(line)
-      self.class.new(path, input).add_state(line)
-    end
   end
 
   class FlushableListener < FileListener

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.17'
+  s.version         = '4.2.3'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -31,6 +31,7 @@ Gem::Specification.new do |s|
     s.add_runtime_dependency 'addressable'
   end
 
+  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
   s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']

data/spec/filewatch/reading_spec.rb

@@ -23,9 +23,12 @@ module FileWatch
     let(:start_new_files_at) { :end } # should be irrelevant for read mode
     let(:opts) do
       {
-        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at,
-        :delimiter => "\n", :discover_interval => discover_interval,
-        :ignore_older => 3600, :sincedb_path => sincedb_path
+        :stat_interval => stat_interval,
+        :start_new_files_at => start_new_files_at,
+        :delimiter => "\n",
+        :discover_interval => discover_interval,
+        :ignore_older => 3600,
+        :sincedb_path => sincedb_path
       }
     end
     let(:observer) { TestObserver.new }
@@ -147,6 +150,50 @@ module FileWatch
       end
     end
 
+    context "when watching directory with files and adding a new file" do
+      let(:file_path2) { ::File.join(directory, "2.log") }
+      let(:file_path3) { ::File.join(directory, "3.log") }
+
+      let(:opts) { super.merge(:file_sort_by => "last_modified") }
+      let(:lines) { [] }
+      let(:observer) { TestObserver.new(lines) }
+
+
+      let(:listener2) { observer.listener_for(file_path2) }
+      let(:listener3) { observer.listener_for(file_path3) }
+
+      let(:actions) do
+        RSpec::Sequencing.run("create12") do
+          File.open(file_path,  "w") { |file| file.write("string11\nstring12") }
+          File.open(file_path2, "w") { |file| file.write("string21\nstring22") }
+        end
+        .then("watch") do
+          reading.watch_this(watch_dir)
+        end
+        .then("wait12") do
+          wait(2).for { listener1.calls.last == :delete && listener2.calls.last == :delete }.to eq(true)
+        end
+        .then_after(2, "create3") do
+          File.open(file_path3, "w") { |file| file.write("string31\nstring32") }
+        end
+        .then("wait3") do
+          wait(2).for { listener3.calls.last == :delete }.to eq(true)
+        end
+        .then("quit") do
+          reading.quit
+        end
+      end
+
+      it "reads all (3) files" do
+        actions.activate_quietly
+        reading.subscribe(observer)
+        actions.assert_no_errors
+        expect(lines.last).to eq 'string32'
+        expect(lines.sort).to eq %w(string11 string12 string21 string22 string31 string32)
+        expect( reading.watch.watched_files_collection.paths ).to eq [ file_path, file_path2, file_path3 ]
+      end
+    end
+
     context "when watching a directory with files using exit_after_read" do
       let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) }
       let(:file_path3) { ::File.join(directory, "3.log") }
@@ -159,40 +206,45 @@ module FileWatch
       let(:listener6) { observer.listener_for(file_path6) }
 
       it "the file is read" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines).to eq(["line1", "line2"])
       end
+
       it "multiple files are read" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4"])
       end
+
       it "multiple files are read even if max_open_files is smaller then number of files" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
         File.open(file_path5, "w") { |file| file.write("line5\nline6\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4", "line5", "line6"])
       end
+
       it "file as marked as reading_completed" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
       end
+
       it "sincedb works correctly" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         sincedb_record_fields = File.read(sincedb_path).split(" ")
         position_field_index = 3
         expect(sincedb_record_fields[position_field_index]).to eq("12")
       end
+
       it "does not include new files added after start" do
         File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
@@ -201,7 +253,6 @@ module FileWatch
         expect(listener3.lines).to eq(["line1", "line2"])
         expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
         expect(listener6.calls).to eq([])
-
       end
     
     end

data/spec/filewatch/settings_spec.rb

@@ -1,3 +1,6 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/friendly_durations'
+
 describe FileWatch::Settings do
 
   context "when create from options" do

data/spec/filewatch/sincedb_record_serializer_spec.rb

@@ -9,7 +9,9 @@ module FileWatch
     let(:io) { StringIO.new }
     let(:db) { Hash.new }
 
-    subject { SincedbRecordSerializer.new(SincedbRecordSerializer.days_to_seconds(14)) }
+    let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(14) }
+
+    subject { SincedbRecordSerializer.new(sincedb_value_expiry) }
 
     context "deserialize from IO" do
       it 'reads V1 records' do
@@ -82,8 +84,10 @@ module FileWatch
     end
 
     context "given a non default `sincedb_clean_after`" do
+
+      let(:sincedb_value_expiry) { SincedbRecordSerializer.days_to_seconds(2) }
+
       it "does not write expired db entries to an IO object" do
-        subject.update_sincedb_value_expiry_from_days(2)
         one_day_ago = Time.now.to_f - (1.0*24*3600)
         three_days_ago = one_day_ago - (2.0*24*3600)
         db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)

data/spec/filewatch/spec_helper.rb

@@ -117,17 +117,12 @@ module FileWatch
     class Listener
       attr_reader :path, :lines, :calls
 
-      def initialize(path)
+      def initialize(path, lines)
         @path = path
-        @lines = Concurrent::Array.new
+        @lines = lines || Concurrent::Array.new
         @calls = Concurrent::Array.new
       end
 
-      def add_lines(lines)
-        @lines = lines
-        self
-      end
-
       def accept(line)
         @lines << line
         @calls << :accept
@@ -161,12 +156,7 @@ module FileWatch
     attr_reader :listeners
 
     def initialize(combined_lines = nil)
-      listener_proc = if combined_lines.nil?
-        lambda{|k| Listener.new(k) }
-      else
-        lambda{|k| Listener.new(k).add_lines(combined_lines) }
-      end
-      @listeners = Concurrent::Hash.new {|hash, key| hash[key] = listener_proc.call(key) }
+      @listeners = Concurrent::Hash.new { |hash, key| hash[key] = new_listener(key, combined_lines) }
     end
 
     def listener_for(path)
@@ -174,6 +164,14 @@ module FileWatch
     end
 
     def clear
-      @listeners.clear; end
+      @listeners.clear
+    end
+
+    private
+
+    def new_listener(path, lines = nil)
+      Listener.new(path, lines)
+    end
+
   end
 end

data/spec/filewatch/tailing_spec.rb

@@ -10,15 +10,19 @@ module FileWatch
     let(:file_path)  { ::File.join(directory, "1#{suffix}.log") }
     let(:file_path2) { ::File.join(directory, "2#{suffix}.log") }
     let(:file_path3) { ::File.join(directory, "3#{suffix}.log") }
-    let(:max)   { 4095 }
+    let(:max) { 4095 }
     let(:stat_interval) { 0.1 }
     let(:discover_interval) { 4 }
     let(:start_new_files_at) { :end }
     let(:sincedb_path) { ::File.join(directory, "tailing.sdb") }
     let(:opts) do
       {
-        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_open_files => max,
-        :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path,
+        :stat_interval => stat_interval,
+        :start_new_files_at => start_new_files_at,
+        :max_open_files => max,
+        :delimiter => "\n",
+        :discover_interval => discover_interval,
+        :sincedb_path => sincedb_path,
         :file_sort_by => "path"
       }
     end
@@ -30,12 +34,11 @@ module FileWatch
 
     before do
       directory
-      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      wait(1.0).for { Dir.exist?(directory) }.to eq(true)
     end
 
     after do
       FileUtils.rm_rf(directory)
-      wait(1.0).for{Dir.exist?(directory)}.to eq(false)
     end
 
     describe "max open files (set to 1)" do
@@ -95,16 +98,16 @@ module FileWatch
       let(:actions) do
         RSpec::Sequencing
           .run("create file") do
-            File.open(file_path, "wb") { |file|  file.write("lineA\nlineB\n") }
+            File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") }
           end
           .then_after(0.1, "begin watching") do
             tailing.watch_this(watch_dir)
           end
-          .then_after(2, "add content") do
-            File.open(file_path, "ab") { |file|  file.write("line1\nline2\n") }
+          .then_after(1.0, "add content") do
+            File.open(file_path, "ab") { |file| file.write("line1\nline2\n") }
           end
           .then("wait") do
-            wait(0.75).for{listener1.lines}.to eq(["line1", "line2"])
+            wait(0.75).for { listener1.lines }.to_not be_empty
           end
           .then("quit") do
             tailing.quit
@@ -113,7 +116,6 @@ module FileWatch
 
       it "only the new content is read" do
         actions.activate_quietly
-        tailing.watch_this(watch_dir)
         tailing.subscribe(observer)
         actions.assert_no_errors
         expect(listener1.calls).to eq([:open, :accept, :accept])
@@ -132,7 +134,7 @@ module FileWatch
             File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
           end
           .then("wait") do
-            wait(0.75).for{listener1.lines.size}.to eq(2)
+            wait(0.75).for { listener1.lines }.to_not be_empty
           end
           .then("quit") do
             tailing.quit
@@ -154,7 +156,7 @@ module FileWatch
       # so when a stat is taken on the file an error is raised
       let(:suffix) { "E" }
       let(:quit_after) { 0.2 }
-      let(:stat)  { double("stat", :size => 100, :modified_at => Time.now.to_f, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) }
+      let(:stat)  { double("stat", :size => 100, :modified_at => Time.now.to_f, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) }
       let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) }
       before do
         allow(stat).to receive(:restat).and_raise(Errno::ENOENT)