logstash-input-file 4.1.16 → 4.2.2

data/lib/logstash/inputs/delete_completed_file_handler.rb

@@ -2,8 +2,13 @@
 
 module LogStash module Inputs
   class DeleteCompletedFileHandler
+    def initialize(watch)
+      @watch = watch
+    end
+
     def handle(path)
       Pathname.new(path).unlink rescue nil
+      @watch.watched_files_collection.remove_paths([path])
     end
   end
 end end

data/lib/logstash/inputs/file.rb

@@ -6,6 +6,7 @@ require "logstash/codecs/identity_map_codec"
 require "pathname"
 require "socket" # for Socket.gethostname
 require "fileutils"
+require "concurrent/atomic/atomic_reference"
 
 require_relative "file/patch"
 require_relative "file_listener"
@@ -227,6 +228,11 @@ class File < LogStash::Inputs::Base
   # Sincedb still works, if you run LS once again after doing some changes - only new values will be read
   config :exit_after_read, :validate => :boolean, :default => false
 
+  # Before start read a compressed file, checks for its validity.
+  # This request a full read of the archive, so potentially could cost time.
+  # If not specified to true, and the file is corrupted, could end in cyclic processing of the broken file.
+  config :check_archive_validity, :validate => :boolean, :default => false
+
   public
 
   class << self
@@ -242,6 +248,9 @@ class File < LogStash::Inputs::Base
     end
   end
 
+  # @private used in specs
+  attr_reader :watcher
+
   def register
     require "addressable/uri"
     require "digest/md5"
@@ -266,10 +275,9 @@ class File < LogStash::Inputs::Base
       :file_sort_by => @file_sort_by,
       :file_sort_direction => @file_sort_direction,
       :exit_after_read => @exit_after_read,
+      :check_archive_validity => @check_archive_validity,
     }
 
-    @completed_file_handlers = []
-
     @path.each do |path|
       if Pathname.new(path).relative?
         raise ArgumentError.new("File paths must be absolute, relative path specified: #{path}")
@@ -313,15 +321,10 @@ class File < LogStash::Inputs::Base
       @watcher_class = FileWatch::ObservingTail
     else
       @watcher_class = FileWatch::ObservingRead
-      if @file_completed_action.include?('log')
-        @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path)
-      end
-      if @file_completed_action.include?('delete')
-        @completed_file_handlers << DeleteCompletedFileHandler.new
-      end
     end
     @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
     @completely_stopped = Concurrent::AtomicBoolean.new
+    @queue = Concurrent::AtomicReference.new
   end # def register
 
   def completely_stopped?
@@ -329,8 +332,9 @@ class File < LogStash::Inputs::Base
     @completely_stopped.true?
   end
 
+  # The WatchedFile calls back here as `observer.listener_for(@path)`
+  # @param [String] path the identity
   def listener_for(path)
-    # path is the identity
     FileListener.new(path, self)
   end
 
@@ -338,13 +342,25 @@ class File < LogStash::Inputs::Base
     # if the pipeline restarts this input,
     # make sure previous files are closed
     stop
+
     @watcher = @watcher_class.new(@filewatch_config)
+
+    @completed_file_handlers = []
+    if read_mode?
+      if @file_completed_action.include?('log')
+        @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path)
+      end
+      if @file_completed_action.include?('delete')
+        @completed_file_handlers << DeleteCompletedFileHandler.new(@watcher.watch)
+      end
+    end
+
     @path.each { |path| @watcher.watch_this(path) }
   end
 
   def run(queue)
     start_processing
-    @queue = queue
+    @queue.set queue
     @watcher.subscribe(self) # halts here until quit is called
     # last action of the subscribe call is to write the sincedb
     exit_flush
@@ -355,7 +371,7 @@ class File < LogStash::Inputs::Base
     event.set("[@metadata][host]", @host)
     event.set("host", @host) unless event.include?("host")
     decorate(event)
-    @queue << event
+    @queue.get << event
   end
 
   def handle_deletable_path(path)
@@ -376,6 +392,11 @@ class File < LogStash::Inputs::Base
     end
   end
 
+  # @private used in specs
+  def queue
+    @queue.get
+  end
+
   private
 
   def build_sincedb_base_from_settings(settings)

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.16'
+  s.version         = '4.2.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -31,10 +31,11 @@ Gem::Specification.new do |s|
     s.add_runtime_dependency 'addressable'
   end
 
+  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
   s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
-  s.add_development_dependency 'logstash-devutils', '~> 1.3'
+  s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'rspec-sequencing'
   s.add_development_dependency "rspec-wait"

data/spec/filewatch/reading_spec.rb

@@ -23,9 +23,12 @@ module FileWatch
     let(:start_new_files_at) { :end } # should be irrelevant for read mode
     let(:opts) do
       {
-        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at,
-        :delimiter => "\n", :discover_interval => discover_interval,
-        :ignore_older => 3600, :sincedb_path => sincedb_path
+        :stat_interval => stat_interval,
+        :start_new_files_at => start_new_files_at,
+        :delimiter => "\n",
+        :discover_interval => discover_interval,
+        :ignore_older => 3600,
+        :sincedb_path => sincedb_path
       }
     end
     let(:observer) { TestObserver.new }
@@ -147,6 +150,50 @@ module FileWatch
       end
     end
 
+    context "when watching directory with files and adding a new file" do
+      let(:file_path2) { ::File.join(directory, "2.log") }
+      let(:file_path3) { ::File.join(directory, "3.log") }
+
+      let(:opts) { super.merge(:file_sort_by => "last_modified") }
+      let(:lines) { [] }
+      let(:observer) { TestObserver.new(lines) }
+
+
+      let(:listener2) { observer.listener_for(file_path2) }
+      let(:listener3) { observer.listener_for(file_path3) }
+
+      let(:actions) do
+        RSpec::Sequencing.run("create12") do
+          File.open(file_path,  "w") { |file| file.write("string11\nstring12") }
+          File.open(file_path2, "w") { |file| file.write("string21\nstring22") }
+        end
+        .then("watch") do
+          reading.watch_this(watch_dir)
+        end
+        .then("wait12") do
+          wait(2).for { listener1.calls.last == :delete && listener2.calls.last == :delete }.to eq(true)
+        end
+        .then_after(2, "create3") do
+          File.open(file_path3, "w") { |file| file.write("string31\nstring32") }
+        end
+        .then("wait3") do
+          wait(2).for { listener3.calls.last == :delete }.to eq(true)
+        end
+        .then("quit") do
+          reading.quit
+        end
+      end
+
+      it "reads all (3) files" do
+        actions.activate_quietly
+        reading.subscribe(observer)
+        actions.assert_no_errors
+        expect(lines.last).to eq 'string32'
+        expect(lines.sort).to eq %w(string11 string12 string21 string22 string31 string32)
+        expect( reading.watch.watched_files_collection.paths ).to eq [ file_path, file_path2, file_path3 ]
+      end
+    end
+
     context "when watching a directory with files using exit_after_read" do
       let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) }
       let(:file_path3) { ::File.join(directory, "3.log") }
@@ -159,40 +206,45 @@ module FileWatch
       let(:listener6) { observer.listener_for(file_path6) }
 
       it "the file is read" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines).to eq(["line1", "line2"])
       end
+
       it "multiple files are read" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4"])
       end
+
       it "multiple files are read even if max_open_files is smaller then number of files" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
         File.open(file_path5, "w") { |file| file.write("line5\nline6\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4", "line5", "line6"])
       end
+
       it "file as marked as reading_completed" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
       end
+
       it "sincedb works correctly" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         sincedb_record_fields = File.read(sincedb_path).split(" ")
         position_field_index = 3
         expect(sincedb_record_fields[position_field_index]).to eq("12")
       end
+
       it "does not include new files added after start" do
         File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
@@ -201,7 +253,6 @@ module FileWatch
         expect(listener3.lines).to eq(["line1", "line2"])
         expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
         expect(listener6.calls).to eq([])
-
       end
     
     end

data/spec/filewatch/rotate_spec.rb

@@ -73,7 +73,8 @@ module FileWatch
             FileUtils.mv(directory.join("1.logtmp").to_path, file1_path)
           end
           .then("wait for expectation") do
-            wait(2).for{listener1.calls}.to eq([:open, :accept, :accept, :accept])
+            sleep(0.25) # if ENV['CI']
+            wait(2).for { listener1.calls }.to eq([:open, :accept, :accept, :accept])
           end
           .then("quit") do
             tailing.quit

data/spec/filewatch/settings_spec.rb

@@ -1,3 +1,6 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/friendly_durations'
+
 describe FileWatch::Settings do
 
   context "when create from options" do

data/spec/filewatch/spec_helper.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require "rspec_sequencing"
-# require 'rspec/wait'
+require 'rspec/wait'
 require "logstash/devutils/rspec/spec_helper"
 require "concurrent"
 require "timecop"
@@ -117,17 +117,12 @@ module FileWatch
     class Listener
       attr_reader :path, :lines, :calls
 
-      def initialize(path)
+      def initialize(path, lines)
         @path = path
-        @lines = Concurrent::Array.new
+        @lines = lines || Concurrent::Array.new
         @calls = Concurrent::Array.new
       end
 
-      def add_lines(lines)
-        @lines = lines
-        self
-      end
-
       def accept(line)
         @lines << line
         @calls << :accept
@@ -161,12 +156,7 @@ module FileWatch
     attr_reader :listeners
 
     def initialize(combined_lines = nil)
-      listener_proc = if combined_lines.nil?
-        lambda{|k| Listener.new(k) }
-      else
-        lambda{|k| Listener.new(k).add_lines(combined_lines) }
-      end
-      @listeners = Concurrent::Hash.new {|hash, key| hash[key] = listener_proc.call(key) }
+      @listeners = Concurrent::Hash.new { |hash, key| hash[key] = new_listener(key, combined_lines) }
     end
 
     def listener_for(path)
@@ -174,6 +164,14 @@ module FileWatch
     end
 
     def clear
-      @listeners.clear; end
+      @listeners.clear
+    end
+
+    private
+
+    def new_listener(path, lines = nil)
+      Listener.new(path, lines)
+    end
+
   end
 end

data/spec/filewatch/tailing_spec.rb

@@ -10,15 +10,19 @@ module FileWatch
     let(:file_path)  { ::File.join(directory, "1#{suffix}.log") }
     let(:file_path2) { ::File.join(directory, "2#{suffix}.log") }
     let(:file_path3) { ::File.join(directory, "3#{suffix}.log") }
-    let(:max)   { 4095 }
+    let(:max) { 4095 }
     let(:stat_interval) { 0.1 }
     let(:discover_interval) { 4 }
     let(:start_new_files_at) { :end }
     let(:sincedb_path) { ::File.join(directory, "tailing.sdb") }
     let(:opts) do
       {
-        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_open_files => max,
-        :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path,
+        :stat_interval => stat_interval,
+        :start_new_files_at => start_new_files_at,
+        :max_open_files => max,
+        :delimiter => "\n",
+        :discover_interval => discover_interval,
+        :sincedb_path => sincedb_path,
         :file_sort_by => "path"
       }
     end
@@ -30,12 +34,11 @@ module FileWatch
 
     before do
       directory
-      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      wait(1.0).for { Dir.exist?(directory) }.to eq(true)
     end
 
     after do
       FileUtils.rm_rf(directory)
-      wait(1.0).for{Dir.exist?(directory)}.to eq(false)
     end
 
     describe "max open files (set to 1)" do
@@ -95,16 +98,16 @@ module FileWatch
       let(:actions) do
         RSpec::Sequencing
           .run("create file") do
-            File.open(file_path, "wb") { |file|  file.write("lineA\nlineB\n") }
+            File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") }
           end
           .then_after(0.1, "begin watching") do
             tailing.watch_this(watch_dir)
           end
-          .then_after(2, "add content") do
-            File.open(file_path, "ab") { |file|  file.write("line1\nline2\n") }
+          .then_after(1.0, "add content") do
+            File.open(file_path, "ab") { |file| file.write("line1\nline2\n") }
           end
           .then("wait") do
-            wait(0.75).for{listener1.lines}.to eq(["line1", "line2"])
+            wait(0.75).for { listener1.lines }.to_not be_empty
           end
           .then("quit") do
             tailing.quit
@@ -113,7 +116,6 @@ module FileWatch
 
       it "only the new content is read" do
         actions.activate_quietly
-        tailing.watch_this(watch_dir)
         tailing.subscribe(observer)
         actions.assert_no_errors
         expect(listener1.calls).to eq([:open, :accept, :accept])
@@ -132,7 +134,7 @@ module FileWatch
             File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
           end
           .then("wait") do
-            wait(0.75).for{listener1.lines.size}.to eq(2)
+            wait(0.75).for { listener1.lines }.to_not be_empty
           end
           .then("quit") do
             tailing.quit
@@ -154,7 +156,7 @@ module FileWatch
       # so when a stat is taken on the file an error is raised
       let(:suffix) { "E" }
       let(:quit_after) { 0.2 }
-      let(:stat)  { double("stat", :size => 100, :modified_at => Time.now.to_f, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) }
+      let(:stat)  { double("stat", :size => 100, :modified_at => Time.now.to_f, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) }
       let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) }
       before do
         allow(stat).to receive(:restat).and_raise(Errno::ENOENT)

data/spec/filewatch/watched_file_spec.rb

@@ -35,5 +35,35 @@ module FileWatch
         expect(watched_file.recent_states).to eq([:watched, :active, :watched, :closed, :watched, :active, :unwatched, :active])
       end
     end
+
+    context 'restat' do
+
+      let(:directory) { Stud::Temporary.directory }
+      let(:file_path) { ::File.join(directory, "restat.file.txt") }
+      let(:pathname) { Pathname.new(file_path) }
+
+      before { FileUtils.touch file_path, :mtime => Time.now - 300 }
+
+      it 'reports false value when no changes' do
+        file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new)
+        mtime = file.modified_at
+        expect( file.modified_at_changed? ).to be false
+        expect( file.restat! ).to be_falsy
+        expect( file.modified_at_changed? ).to be false
+        expect( file.modified_at ).to eql mtime
+        expect( file.modified_at(true) ).to eql mtime
+      end
+
+      it 'reports truthy when changes detected' do
+        file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new)
+        mtime = file.modified_at
+        expect( file.modified_at_changed? ).to be false
+        FileUtils.touch file_path
+        expect( file.restat! ).to be_truthy
+        expect( file.modified_at_changed? ).to be true
+        expect( file.modified_at ).to eql mtime # until updated
+        expect( file.modified_at(true) ).to be > mtime
+      end
+    end
   end
 end