logstash-input-file 4.1.17 → 4.1.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cdb425f23de3ef4ccd1a1358ce48789001145a9a4620405bac4dc6a75d2b7f69
-  data.tar.gz: b7c856e75936d99912a8a517b81e23be1603e4032a255e6340b2c4da7faa6639
+  metadata.gz: dc2b7f39bcc8e3dfb894ea72af1b07e78fff73bf97ad12614a9ec440fed0c2d1
+  data.tar.gz: 6715064a8a25a13538cc07216f5c479f79dd2d3ba73a267377cd357c6e27ea8d
 SHA512:
-  metadata.gz: d0641b7f747964983a6f2e3b8c68a19efc74bb085e6a635cc110cd8d1c43efe330a899a951933f228cf59015785cc00730bd196a84a63dd36241045d36614ebf
-  data.tar.gz: 834e9ad07ed15beaee47e0cd3d6d6f5e5494e41a6d5a2f4871f79df4d63837ef3c480f033eaac0d092794f4c0a36fe4d3466728d98cf2ad55ac564cd8dec0f4f
+  metadata.gz: fbc9d7cd0f7b50be21721bfe0cb7c26a1e87a1f5697f4a01f14dcdc14a8449c9a9d14ddec58f3ec1b8bd611e624855b538155af8f2924604fb9f1cf120de4e22
+  data.tar.gz: 44b4bdb7b356a6494cb1cde6bf48395d61222234d2ec6cfb134d551a20ed1ab08043b23266b5bc75854b85577d22a1fd36d5b40c17213eb10df68f2903086146

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.18
+  - Fix: release watched files on completion (in read-mode) [#271](https://github.com/logstash-plugins/logstash-input-file/pull/271)
+
 ## 4.1.17
   - Added configuration setting `check_archive_validity` settings to enable
   gzipped files verification, issue

data/lib/filewatch/read_mode/handlers/read_file.rb

@@ -22,6 +22,9 @@ module FileWatch module ReadMode module Handlers
           sincedb_collection.reading_completed(key)
           sincedb_collection.clear_watched_file(key)
           watched_file.listener.deleted
+          # NOTE: on top of un-watching we should also remove from the watched files collection
+          # if the file is getting deleted (on completion), that part currently resides in
+          # DeleteCompletedFileHandler - triggered above using `watched_file.listener.deleted`
           watched_file.unwatch
         end
       end

data/lib/filewatch/watch.rb

@@ -67,7 +67,7 @@ module FileWatch
         watched_files = @watched_files_collection.values
         @processor.process_all_states(watched_files)
       ensure
-        @watched_files_collection.delete(@processor.deletable_filepaths)
+        @watched_files_collection.remove_paths(@processor.deletable_filepaths)
         @processor.deletable_filepaths.clear
       end
     end # def each

data/lib/filewatch/watched_files_collection.rb

@@ -15,13 +15,17 @@ module FileWatch
       @sort_method.call
     end
 
-    def delete(paths)
-      Array(paths).each do |f|
-        index = @pointers.delete(f)
-        @files.delete_at(index)
-        refresh_pointers
+    def remove_paths(paths)
+      removed_files = Array(paths).map do |path|
+        index = @pointers.delete(path)
+        if index
+          watched_file = @files.delete_at(index)
+          refresh_pointers
+          watched_file
+        end
       end
       @sort_method.call
+      removed_files
     end
 
     def close_all

data/lib/logstash/inputs/delete_completed_file_handler.rb

@@ -2,8 +2,13 @@
 
 module LogStash module Inputs
   class DeleteCompletedFileHandler
+    def initialize(watch)
+      @watch = watch
+    end
+
     def handle(path)
       Pathname.new(path).unlink rescue nil
+      @watch.watched_files_collection.remove_paths([path])
     end
   end
 end end

data/lib/logstash/inputs/file.rb

@@ -6,6 +6,7 @@ require "logstash/codecs/identity_map_codec"
 require "pathname"
 require "socket" # for Socket.gethostname
 require "fileutils"
+require "concurrent/atomic/atomic_reference"
 
 require_relative "file/patch"
 require_relative "file_listener"
@@ -247,6 +248,9 @@ class File < LogStash::Inputs::Base
     end
   end
 
+  # @private used in specs
+  attr_reader :watcher
+
   def register
     require "addressable/uri"
     require "digest/md5"
@@ -274,8 +278,6 @@ class File < LogStash::Inputs::Base
       :check_archive_validity => @check_archive_validity,
     }
 
-    @completed_file_handlers = []
-
     @path.each do |path|
       if Pathname.new(path).relative?
         raise ArgumentError.new("File paths must be absolute, relative path specified: #{path}")
@@ -319,15 +321,10 @@ class File < LogStash::Inputs::Base
       @watcher_class = FileWatch::ObservingTail
     else
       @watcher_class = FileWatch::ObservingRead
-      if @file_completed_action.include?('log')
-        @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path)
-      end
-      if @file_completed_action.include?('delete')
-        @completed_file_handlers << DeleteCompletedFileHandler.new
-      end
     end
     @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
     @completely_stopped = Concurrent::AtomicBoolean.new
+    @queue = Concurrent::AtomicReference.new
   end # def register
 
   def completely_stopped?
@@ -344,13 +341,25 @@ class File < LogStash::Inputs::Base
     # if the pipeline restarts this input,
     # make sure previous files are closed
     stop
+
     @watcher = @watcher_class.new(@filewatch_config)
+
+    @completed_file_handlers = []
+    if read_mode?
+      if @file_completed_action.include?('log')
+        @completed_file_handlers << LogCompletedFileHandler.new(@file_completed_log_path)
+      end
+      if @file_completed_action.include?('delete')
+        @completed_file_handlers << DeleteCompletedFileHandler.new(@watcher.watch)
+      end
+    end
+
     @path.each { |path| @watcher.watch_this(path) }
   end
 
   def run(queue)
     start_processing
-    @queue = queue
+    @queue.set queue
     @watcher.subscribe(self) # halts here until quit is called
     # last action of the subscribe call is to write the sincedb
     exit_flush
@@ -361,7 +370,7 @@ class File < LogStash::Inputs::Base
     event.set("[@metadata][host]", @host)
     event.set("host", @host) unless event.include?("host")
     decorate(event)
-    @queue << event
+    @queue.get << event
   end
 
   def handle_deletable_path(path)
@@ -382,6 +391,11 @@ class File < LogStash::Inputs::Base
     end
   end
 
+  # @private used in specs
+  def queue
+    @queue.get
+  end
+
   private
 
   def build_sincedb_base_from_settings(settings)

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.17'
+  s.version         = '4.1.18'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filewatch/watched_files_collection_spec.rb

@@ -85,9 +85,9 @@ module FileWatch
         collection.add(wf3)
         expect(collection.keys).to eq([filepath1, filepath2, filepath3])
 
-        collection.delete([filepath2,filepath3])
+        collection.remove_paths([filepath2,filepath3])
         expect(collection.keys).to eq([filepath1])
-
+        expect(collection.values.size).to eq 1
       end
     end
 

data/spec/helpers/spec_helper.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require "logstash/devutils/rspec/spec_helper"
+require "rspec/wait"
 require "rspec_sequencing"
 
 module FileInput

data/spec/inputs/file_read_spec.rb

@@ -247,4 +247,79 @@ describe LogStash::Inputs::File do
       end
     end
   end
+
+  let(:temp_directory) { Stud::Temporary.directory }
+  let(:interval) { 0.1 }
+  let(:options) do
+    {
+        'mode' => "read",
+        'path' => "#{temp_directory}/*",
+        'stat_interval' => interval,
+        'discover_interval' => interval,
+        'sincedb_path' => "#{temp_directory}/.sincedb",
+        'sincedb_write_interval' => interval
+    }
+  end
+
+  let(:queue) { Queue.new }
+  let(:plugin) { LogStash::Inputs::File.new(options) }
+
+  describe 'delete on complete' do
+
+    let(:options) do
+      super.merge({ 'file_completed_action' => "delete", 'exit_after_read' => false })
+    end
+
+    let(:sample_file) { File.join(temp_directory, "sample.log") }
+
+    before do
+      plugin.register
+      @run_thread = Thread.new(plugin) do |plugin|
+        Thread.current.abort_on_exception = true
+        plugin.run queue
+      end
+
+      File.open(sample_file, 'w') { |fd| fd.write("sample-content\n") }
+
+      wait_for_start_processing(@run_thread)
+    end
+
+    after { plugin.stop }
+
+    it 'processes a file' do
+      wait_for_file_removal(sample_file) # watched discovery
+
+      expect( plugin.queue.size ).to eql 1
+      event = plugin.queue.pop
+      expect( event.get('message') ).to eql 'sample-content'
+    end
+
+    it 'removes watched file from collection' do
+      wait_for_file_removal(sample_file) # watched discovery
+      sleep(0.25) # give CI some space to execute the removal
+      # TODO shouldn't be necessary once WatchedFileCollection does proper locking
+      watched_files = plugin.watcher.watch.watched_files_collection
+      expect( watched_files ).to be_empty
+    end
+
+    private
+
+    def wait_for_start_processing(run_thread, timeout: 1.0)
+      begin
+        Timeout.timeout(timeout) do
+          sleep(0.01) while run_thread.status != 'sleep'
+          sleep(timeout) unless plugin.queue
+        end
+      rescue Timeout::Error
+        raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue
+      else
+        raise "plugin did not start processing" unless plugin.queue
+      end
+    end
+
+    def wait_for_file_removal(path, timeout: 3 * interval)
+      wait(timeout).for { File.exist?(path) }.to be_falsey
+    end
+
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.1.17
+  version: 4.1.18
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-03 00:00:00.000000000 Z
+date: 2020-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement