logstash-input-file 4.1.17 → 4.2.3

data/spec/filewatch/watched_file_spec.rb

@@ -35,5 +35,35 @@ module FileWatch
         expect(watched_file.recent_states).to eq([:watched, :active, :watched, :closed, :watched, :active, :unwatched, :active])
       end
     end
+
+    context 'restat' do
+
+      let(:directory) { Stud::Temporary.directory }
+      let(:file_path) { ::File.join(directory, "restat.file.txt") }
+      let(:pathname) { Pathname.new(file_path) }
+
+      before { FileUtils.touch file_path, :mtime => Time.now - 300 }
+
+      it 'reports false value when no changes' do
+        file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new)
+        mtime = file.modified_at
+        expect( file.modified_at_changed? ).to be false
+        expect( file.restat! ).to be_falsy
+        expect( file.modified_at_changed? ).to be false
+        expect( file.modified_at ).to eql mtime
+        expect( file.modified_at(true) ).to eql mtime
+      end
+
+      it 'reports truthy when changes detected' do
+        file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new)
+        mtime = file.modified_at
+        expect( file.modified_at_changed? ).to be false
+        FileUtils.touch file_path
+        expect( file.restat! ).to be_truthy
+        expect( file.modified_at_changed? ).to be true
+        expect( file.modified_at ).to eql mtime # until updated
+        expect( file.modified_at(true) ).to be > mtime
+      end
+    end
   end
 end

data/spec/filewatch/watched_files_collection_spec.rb

@@ -4,15 +4,18 @@ require_relative 'spec_helper'
 module FileWatch
   describe WatchedFilesCollection do
     let(:time) { Time.now }
-    let(:filepath1){"/var/log/z.log"}
-    let(:filepath2){"/var/log/m.log"}
-    let(:filepath3){"/var/log/a.log"}
-    let(:stat1)  { double("stat1", :size => 98, :modified_at => time - 30, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
-    let(:stat2)  { double("stat2", :size => 99, :modified_at => time - 20, :identifier => nil, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
-    let(:stat3)  { double("stat3", :size => 100, :modified_at => time, :identifier => nil, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) }
+    let(:filepath1) { "/var/log/z.log" }
+    let(:filepath2) { "/var/log/m.log" }
+    let(:filepath3) { "/var/log/a.log" }
+    let(:filepath4) { "/var/log/b.log" }
+    let(:stat1) { double("stat1", :size => 98, :modified_at => time - 30, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
+    let(:stat2) { double("stat2", :size => 99, :modified_at => time - 20, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
+    let(:stat3) { double("stat3", :size => 100, :modified_at => time, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) }
+    let(:stat4) { double("stat4", :size => 99, :modified_at => time, :inode => 234570, :inode_struct => InodeStruct.new("234570", 3, 2)) }
     let(:wf1) { WatchedFile.new(filepath1, stat1, Settings.new) }
     let(:wf2) { WatchedFile.new(filepath2, stat2, Settings.new) }
     let(:wf3) { WatchedFile.new(filepath3, stat3, Settings.new) }
+    let(:wf4) { WatchedFile.new(filepath4, stat4, Settings.new) }
 
     context "sort by last_modified in ascending order" do
       let(:sort_by) { "last_modified" }
@@ -20,12 +23,29 @@ module FileWatch
 
       it "sorts earliest modified first" do
         collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        expect(collection.empty?).to be true
         collection.add(wf2)
+        expect(collection.empty?).to be false
         expect(collection.values).to eq([wf2])
         collection.add(wf3)
         expect(collection.values).to eq([wf2, wf3])
         collection.add(wf1)
         expect(collection.values).to eq([wf1, wf2, wf3])
+        expect(collection.keys.size).to eq 3
+      end
+
+      it "sorts by path when mtime is same" do
+        collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        expect(collection.size).to eq 0
+        collection.add(wf2)
+        collection.add(wf4)
+        collection.add(wf1)
+        expect(collection.size).to eq 3
+        expect(collection.values).to eq([wf1, wf2, wf4])
+        collection.add(wf3)
+        expect(collection.size).to eq 4
+        expect(collection.values).to eq([wf1, wf2, wf3, wf4])
+        expect(collection.keys.size).to eq 4
       end
     end
 
@@ -74,7 +94,7 @@ module FileWatch
       end
     end
 
-    context "when delete called" do
+    context "remove_paths" do
       let(:sort_by) { "path" }
       let(:sort_direction) { "desc" }
 
@@ -85,9 +105,43 @@ module FileWatch
         collection.add(wf3)
         expect(collection.keys).to eq([filepath1, filepath2, filepath3])
 
-        collection.delete([filepath2,filepath3])
+        ret = collection.remove_paths([filepath2, filepath3])
+        expect(ret).to eq 2
         expect(collection.keys).to eq([filepath1])
+        expect(collection.values.size).to eq 1
+
+        ret = collection.remove_paths([filepath2])
+        expect(ret).to eq 0
+      end
+    end
+
+    context "update" do
+      let(:sort_by) { "last_modified" }
+      let(:sort_direction) { "asc" }
+
+      let(:re_stat1) { double("restat1", :size => 99, :modified_at => time, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
+      let(:re_stat2) { double("restat2", :size => 99, :modified_at => time, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
+
+      it "updates entry with changed mtime" do
+        collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        collection.add(wf1)
+        collection.add(wf2)
+        collection.add(wf3)
+        expect(collection.files).to eq([wf1, wf2, wf3])
+
+        wf2.send(:set_stat, re_stat2)
+        expect( wf2.modified_at_changed? ).to be_truthy
+
+        collection.update wf2
+        expect(collection.files).to eq([wf1, wf3, wf2])
+
+        wf1.send(:set_stat, re_stat1)
+        expect( wf1.modified_at_changed? ).to be_truthy
+        collection.update wf1
+        expect(collection.files).to eq([wf3, wf2, wf1])
 
+        collection.add(wf4)
+        expect(collection.files).to eq([wf3, wf4, wf2, wf1])
       end
     end
 

data/spec/helpers/spec_helper.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require "logstash/devutils/rspec/spec_helper"
+require "rspec/wait"
 require "rspec_sequencing"
 
 module FileInput

data/spec/inputs/file_read_spec.rb

@@ -247,4 +247,123 @@ describe LogStash::Inputs::File do
       end
     end
   end
+
+  let(:temp_directory) { Stud::Temporary.directory }
+  let(:interval) { 0.1 }
+  let(:options) do
+    {
+        'mode' => "read",
+        'path' => "#{temp_directory}/*",
+        'stat_interval' => interval,
+        'discover_interval' => interval,
+        'sincedb_path' => "#{temp_directory}/.sincedb",
+        'sincedb_write_interval' => interval
+    }
+  end
+
+  let(:queue) { Queue.new }
+  let(:plugin) { LogStash::Inputs::File.new(options) }
+
+  describe 'delete on complete' do
+
+    let(:options) do
+      super.merge({ 'file_completed_action' => "delete", 'exit_after_read' => false })
+    end
+
+    let(:sample_file) { File.join(temp_directory, "sample.log") }
+
+    before do
+      plugin.register
+      @run_thread = Thread.new(plugin) do |plugin|
+        Thread.current.abort_on_exception = true
+        plugin.run queue
+      end
+
+      File.open(sample_file, 'w') { |fd| fd.write("sample-content\n") }
+
+      wait_for_start_processing(@run_thread)
+    end
+
+    after { plugin.stop }
+
+    it 'processes a file' do
+      wait_for_file_removal(sample_file) # watched discovery
+
+      expect( plugin.queue.size ).to eql 1
+      event = plugin.queue.pop
+      expect( event.get('message') ).to eql 'sample-content'
+    end
+
+    it 'removes watched file from collection' do
+      wait_for_file_removal(sample_file) # watched discovery
+      sleep(0.25) # give CI some space to execute the removal
+      # TODO shouldn't be necessary once WatchedFileCollection does proper locking
+      watched_files = plugin.watcher.watch.watched_files_collection
+      expect( watched_files ).to be_empty
+    end
+  end
+
+  describe 'sincedb cleanup' do
+
+    let(:options) do
+      super.merge(
+          'sincedb_path' => sincedb_path,
+          'sincedb_clean_after' => '1.0 seconds',
+          'sincedb_write_interval' => 0.25,
+          'stat_interval' => 0.1,
+      )
+    end
+
+    let(:sincedb_path) { "#{temp_directory}/.sincedb" }
+
+    let(:sample_file) { File.join(temp_directory, "sample.txt") }
+
+    before do
+      plugin.register
+      @run_thread = Thread.new(plugin) do |plugin|
+        Thread.current.abort_on_exception = true
+        plugin.run queue
+      end
+
+      File.open(sample_file, 'w') { |fd| fd.write("line1\nline2\n") }
+
+      wait_for_start_processing(@run_thread)
+    end
+
+    after { plugin.stop }
+
+    it 'cleans up sincedb entry' do
+      wait_for_file_removal(sample_file) # watched discovery
+
+      sincedb_content = File.read(sincedb_path).strip
+      expect( sincedb_content ).to_not be_empty
+
+      Stud.try(3.times) do
+        sleep(1.5) # > sincedb_clean_after
+
+        sincedb_content = File.read(sincedb_path).strip
+        expect( sincedb_content ).to be_empty
+      end
+    end
+
+  end
+
+  private
+
+  def wait_for_start_processing(run_thread, timeout: 1.0)
+    begin
+      Timeout.timeout(timeout) do
+        sleep(0.01) while run_thread.status != 'sleep'
+        sleep(timeout) unless plugin.queue
+      end
+    rescue Timeout::Error
+      raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue
+    else
+      raise "plugin did not start processing" unless plugin.queue
+    end
+  end
+
+  def wait_for_file_removal(path, timeout: 3 * interval)
+    wait(timeout).for { File.exist?(path) }.to be_falsey
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.1.17
+  version: 4.2.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-03 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,6 +58,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -178,6 +192,7 @@ files:
 - lib/filewatch/observing_base.rb
 - lib/filewatch/observing_read.rb
 - lib/filewatch/observing_tail.rb
+- lib/filewatch/processor.rb
 - lib/filewatch/read_mode/handlers/base.rb
 - lib/filewatch/read_mode/handlers/read_file.rb
 - lib/filewatch/read_mode/handlers/read_zip_file.rb