logstash-input-file 4.1.16 → 4.2.2

data/spec/filewatch/watched_files_collection_spec.rb

@@ -4,15 +4,18 @@ require_relative 'spec_helper'
 module FileWatch
   describe WatchedFilesCollection do
     let(:time) { Time.now }
-    let(:filepath1){"/var/log/z.log"}
-    let(:filepath2){"/var/log/m.log"}
-    let(:filepath3){"/var/log/a.log"}
-    let(:stat1)  { double("stat1", :size => 98, :modified_at => time - 30, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
-    let(:stat2)  { double("stat2", :size => 99, :modified_at => time - 20, :identifier => nil, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
-    let(:stat3)  { double("stat3", :size => 100, :modified_at => time, :identifier => nil, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) }
+    let(:filepath1) { "/var/log/z.log" }
+    let(:filepath2) { "/var/log/m.log" }
+    let(:filepath3) { "/var/log/a.log" }
+    let(:filepath4) { "/var/log/b.log" }
+    let(:stat1) { double("stat1", :size => 98, :modified_at => time - 30, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
+    let(:stat2) { double("stat2", :size => 99, :modified_at => time - 20, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
+    let(:stat3) { double("stat3", :size => 100, :modified_at => time, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) }
+    let(:stat4) { double("stat4", :size => 99, :modified_at => time, :inode => 234570, :inode_struct => InodeStruct.new("234570", 3, 2)) }
     let(:wf1) { WatchedFile.new(filepath1, stat1, Settings.new) }
     let(:wf2) { WatchedFile.new(filepath2, stat2, Settings.new) }
     let(:wf3) { WatchedFile.new(filepath3, stat3, Settings.new) }
+    let(:wf4) { WatchedFile.new(filepath4, stat4, Settings.new) }
 
     context "sort by last_modified in ascending order" do
       let(:sort_by) { "last_modified" }
@@ -20,12 +23,29 @@ module FileWatch
 
       it "sorts earliest modified first" do
         collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        expect(collection.empty?).to be true
         collection.add(wf2)
+        expect(collection.empty?).to be false
         expect(collection.values).to eq([wf2])
         collection.add(wf3)
         expect(collection.values).to eq([wf2, wf3])
         collection.add(wf1)
         expect(collection.values).to eq([wf1, wf2, wf3])
+        expect(collection.keys.size).to eq 3
+      end
+
+      it "sorts by path when mtime is same" do
+        collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        expect(collection.size).to eq 0
+        collection.add(wf2)
+        collection.add(wf4)
+        collection.add(wf1)
+        expect(collection.size).to eq 3
+        expect(collection.values).to eq([wf1, wf2, wf4])
+        collection.add(wf3)
+        expect(collection.size).to eq 4
+        expect(collection.values).to eq([wf1, wf2, wf3, wf4])
+        expect(collection.keys.size).to eq 4
       end
     end
 
@@ -74,7 +94,7 @@ module FileWatch
       end
     end
 
-    context "when delete called" do
+    context "remove_paths" do
       let(:sort_by) { "path" }
       let(:sort_direction) { "desc" }
 
@@ -85,9 +105,43 @@ module FileWatch
         collection.add(wf3)
         expect(collection.keys).to eq([filepath1, filepath2, filepath3])
 
-        collection.delete([filepath2,filepath3])
+        ret = collection.remove_paths([filepath2, filepath3])
+        expect(ret).to eq 2
         expect(collection.keys).to eq([filepath1])
+        expect(collection.values.size).to eq 1
+
+        ret = collection.remove_paths([filepath2])
+        expect(ret).to eq 0
+      end
+    end
+
+    context "update" do
+      let(:sort_by) { "last_modified" }
+      let(:sort_direction) { "asc" }
+
+      let(:re_stat1) { double("restat1", :size => 99, :modified_at => time, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
+      let(:re_stat2) { double("restat2", :size => 99, :modified_at => time, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
+
+      it "updates entry with changed mtime" do
+        collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        collection.add(wf1)
+        collection.add(wf2)
+        collection.add(wf3)
+        expect(collection.files).to eq([wf1, wf2, wf3])
+
+        wf2.send(:set_stat, re_stat2)
+        expect( wf2.modified_at_changed? ).to be_truthy
+
+        collection.update wf2
+        expect(collection.files).to eq([wf1, wf3, wf2])
+
+        wf1.send(:set_stat, re_stat1)
+        expect( wf1.modified_at_changed? ).to be_truthy
+        collection.update wf1
+        expect(collection.files).to eq([wf3, wf2, wf1])
 
+        collection.add(wf4)
+        expect(collection.files).to eq([wf3, wf4, wf2, wf1])
       end
     end
 

data/spec/helpers/spec_helper.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require "logstash/devutils/rspec/spec_helper"
+require "rspec/wait"
 require "rspec_sequencing"
 
 module FileInput
@@ -16,6 +17,13 @@ module FileInput
     ::File.utime(time, time, path)
   end
 
+  def self.corrupt_gzip(file_path)
+    f = File.open(file_path, "w")
+    f.seek(12)
+    f.puts 'corrupting_string'
+    f.close()
+  end
+
   class TracerBase
     def initialize
       @tracer = Concurrent::Array.new

data/spec/inputs/file_read_spec.rb

@@ -70,7 +70,7 @@ describe LogStash::Inputs::File do
       end
 
       events = input(conf) do |pipeline, queue|
-        wait(0.5).for{IO.read(log_completed_path)}.to match(/A\.log/)
+        wait(0.75).for { IO.read(log_completed_path) }.to match(/A\.log/)
         2.times.collect { queue.pop }
       end
       expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world")
@@ -137,7 +137,7 @@ describe LogStash::Inputs::File do
         CONFIG
 
         events = input(conf) do |pipeline, queue|
-          wait(0.5).for{IO.read(log_completed_path)}.to match(/#{file_path.to_s}/)
+          wait(0.75).for { IO.read(log_completed_path) }.to match(/#{file_path.to_s}/)
           2.times.collect { queue.pop }
         end
 
@@ -171,7 +171,7 @@ describe LogStash::Inputs::File do
         CONFIG
 
         events = input(conf) do |pipeline, queue|
-          wait(0.5).for{IO.read(log_completed_path)}.to match(/uncompressed\.log/)
+          wait(0.75).for{ IO.read(log_completed_path) }.to match(/uncompressed\.log/)
           2.times.collect { queue.pop }
         end
 
@@ -205,7 +205,7 @@ describe LogStash::Inputs::File do
         CONFIG
 
         events = input(conf) do |pipeline, queue|
-          wait(0.5).for{IO.read(log_completed_path).scan(/compressed\.log\.gz(ip)?/).size}.to eq(2)
+          wait(0.75).for { IO.read(log_completed_path).scan(/compressed\.log\.gz(ip)?/).size }.to eq(2)
           4.times.collect { queue.pop }
         end
 
@@ -214,6 +214,156 @@ describe LogStash::Inputs::File do
         expect(events[2].get("message")).to start_with("2010-03-12   23:51")
         expect(events[3].get("message")).to start_with("2010-03-12   23:51")
       end
+
+      it "the corrupted file is untouched" do
+        directory = Stud::Temporary.directory
+        file_path = fixture_dir.join('compressed.log.gz')
+        corrupted_file_path = ::File.join(directory, 'corrupted.gz')
+        FileUtils.cp(file_path, corrupted_file_path)
+
+        FileInput.corrupt_gzip(corrupted_file_path)
+
+        log_completed_path = ::File.join(directory, "C_completed.txt")
+        f = File.new(log_completed_path, "w")
+        f.close()
+
+        conf = <<-CONFIG
+        input {
+          file {
+            type => "blah"
+            path => "#{corrupted_file_path}"
+            mode => "read"
+            file_completed_action => "log_and_delete"
+            file_completed_log_path => "#{log_completed_path}"
+            check_archive_validity => true
+          }
+        }
+        CONFIG
+
+        events = input(conf) do |pipeline, queue|
+          wait(1)
+          expect(IO.read(log_completed_path)).to be_empty
+        end
+      end
+    end
+  end
+
+  let(:temp_directory) { Stud::Temporary.directory }
+  let(:interval) { 0.1 }
+  let(:options) do
+    {
+        'mode' => "read",
+        'path' => "#{temp_directory}/*",
+        'stat_interval' => interval,
+        'discover_interval' => interval,
+        'sincedb_path' => "#{temp_directory}/.sincedb",
+        'sincedb_write_interval' => interval
+    }
+  end
+
+  let(:queue) { Queue.new }
+  let(:plugin) { LogStash::Inputs::File.new(options) }
+
+  describe 'delete on complete' do
+
+    let(:options) do
+      super.merge({ 'file_completed_action' => "delete", 'exit_after_read' => false })
+    end
+
+    let(:sample_file) { File.join(temp_directory, "sample.log") }
+
+    before do
+      plugin.register
+      @run_thread = Thread.new(plugin) do |plugin|
+        Thread.current.abort_on_exception = true
+        plugin.run queue
+      end
+
+      File.open(sample_file, 'w') { |fd| fd.write("sample-content\n") }
+
+      wait_for_start_processing(@run_thread)
     end
+
+    after { plugin.stop }
+
+    it 'processes a file' do
+      wait_for_file_removal(sample_file) # watched discovery
+
+      expect( plugin.queue.size ).to eql 1
+      event = plugin.queue.pop
+      expect( event.get('message') ).to eql 'sample-content'
+    end
+
+    it 'removes watched file from collection' do
+      wait_for_file_removal(sample_file) # watched discovery
+      sleep(0.25) # give CI some space to execute the removal
+      # TODO shouldn't be necessary once WatchedFileCollection does proper locking
+      watched_files = plugin.watcher.watch.watched_files_collection
+      expect( watched_files ).to be_empty
+    end
+  end
+
+  describe 'sincedb cleanup' do
+
+    let(:options) do
+      super.merge(
+          'sincedb_path' => sincedb_path,
+          'sincedb_clean_after' => '1.0 seconds',
+          'sincedb_write_interval' => 0.25,
+          'stat_interval' => 0.1,
+      )
+    end
+
+    let(:sincedb_path) { "#{temp_directory}/.sincedb" }
+
+    let(:sample_file) { File.join(temp_directory, "sample.txt") }
+
+    before do
+      plugin.register
+      @run_thread = Thread.new(plugin) do |plugin|
+        Thread.current.abort_on_exception = true
+        plugin.run queue
+      end
+
+      File.open(sample_file, 'w') { |fd| fd.write("line1\nline2\n") }
+
+      wait_for_start_processing(@run_thread)
+    end
+
+    after { plugin.stop }
+
+    it 'cleans up sincedb entry' do
+      wait_for_file_removal(sample_file) # watched discovery
+
+      sincedb_content = File.read(sincedb_path).strip
+      expect( sincedb_content ).to_not be_empty
+
+      Stud.try(3.times) do
+        sleep(1.5) # > sincedb_clean_after
+
+        sincedb_content = File.read(sincedb_path).strip
+        expect( sincedb_content ).to be_empty
+      end
+    end
+
+  end
+
+  private
+
+  def wait_for_start_processing(run_thread, timeout: 1.0)
+    begin
+      Timeout.timeout(timeout) do
+        sleep(0.01) while run_thread.status != 'sleep'
+        sleep(timeout) unless plugin.queue
+      end
+    rescue Timeout::Error
+      raise "plugin did not start processing (timeout: #{timeout})" unless plugin.queue
+    else
+      raise "plugin did not start processing" unless plugin.queue
+    end
+  end
+
+  def wait_for_file_removal(path, timeout: 3 * interval)
+    wait(timeout).for { File.exist?(path) }.to be_falsey
   end
 end

data/spec/inputs/file_tail_spec.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require "helpers/spec_helper"
+require "logstash/devutils/rspec/shared_examples"
 require "logstash/inputs/file"
 
 require "tempfile"
@@ -66,7 +67,7 @@ describe LogStash::Inputs::File do
               path => "#{path_path}"
               start_position => "beginning"
               sincedb_path => "#{sincedb_path}"
-              "file_sort_by" => "path"
+              file_sort_by => "path"
               delimiter => "#{TEST_FILE_DELIMITER}"
             }
           }
@@ -175,7 +176,7 @@ describe LogStash::Inputs::File do
     context "when sincedb_path is a directory" do
       let(:name) { "E" }
       subject { LogStash::Inputs::File.new("path" => path_path, "sincedb_path" => directory) }
-      
+
       after :each do
         FileUtils.rm_rf(sincedb_path)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.1.16
+  version: 4.2.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-20 00:00:00.000000000 Z
+date: 2020-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,6 +58,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -89,17 +103,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -178,6 +192,7 @@ files:
 - lib/filewatch/observing_base.rb
 - lib/filewatch/observing_read.rb
 - lib/filewatch/observing_tail.rb
+- lib/filewatch/processor.rb
 - lib/filewatch/read_mode/handlers/base.rb
 - lib/filewatch/read_mode/handlers/read_file.rb
 - lib/filewatch/read_mode/handlers/read_zip_file.rb