logstash-input-file 4.1.18 → 4.2.0

data/lib/logstash/inputs/file.rb

@@ -332,8 +332,9 @@ class File < LogStash::Inputs::Base
     @completely_stopped.true?
   end
 
+  # The WatchedFile calls back here as `observer.listener_for(@path)`
+  # @param [String] path the identity
   def listener_for(path)
-    # path is the identity
     FileListener.new(path, self)
   end
 

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.1.18'
+  s.version         = '4.2.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -31,6 +31,7 @@ Gem::Specification.new do |s|
     s.add_runtime_dependency 'addressable'
   end
 
+  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0'
   s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']

data/spec/filewatch/reading_spec.rb

@@ -23,9 +23,12 @@ module FileWatch
     let(:start_new_files_at) { :end } # should be irrelevant for read mode
     let(:opts) do
       {
-        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at,
-        :delimiter => "\n", :discover_interval => discover_interval,
-        :ignore_older => 3600, :sincedb_path => sincedb_path
+        :stat_interval => stat_interval,
+        :start_new_files_at => start_new_files_at,
+        :delimiter => "\n",
+        :discover_interval => discover_interval,
+        :ignore_older => 3600,
+        :sincedb_path => sincedb_path
       }
     end
     let(:observer) { TestObserver.new }
@@ -147,6 +150,50 @@ module FileWatch
       end
     end
 
+    context "when watching directory with files and adding a new file" do
+      let(:file_path2) { ::File.join(directory, "2.log") }
+      let(:file_path3) { ::File.join(directory, "3.log") }
+
+      let(:opts) { super.merge(:file_sort_by => "last_modified") }
+      let(:lines) { [] }
+      let(:observer) { TestObserver.new(lines) }
+
+
+      let(:listener2) { observer.listener_for(file_path2) }
+      let(:listener3) { observer.listener_for(file_path3) }
+
+      let(:actions) do
+        RSpec::Sequencing.run("create12") do
+          File.open(file_path,  "w") { |file| file.write("string11\nstring12") }
+          File.open(file_path2, "w") { |file| file.write("string21\nstring22") }
+        end
+        .then("watch") do
+          reading.watch_this(watch_dir)
+        end
+        .then("wait12") do
+          wait(2).for { listener1.calls.last == :delete && listener2.calls.last == :delete }.to eq(true)
+        end
+        .then_after(2, "create3") do
+          File.open(file_path3, "w") { |file| file.write("string31\nstring32") }
+        end
+        .then("wait3") do
+          wait(2).for { listener3.calls.last == :delete }.to eq(true)
+        end
+        .then("quit") do
+          reading.quit
+        end
+      end
+
+      it "reads all (3) files" do
+        actions.activate_quietly
+        reading.subscribe(observer)
+        actions.assert_no_errors
+        expect(lines.last).to eq 'string32'
+        expect(lines.sort).to eq %w(string11 string12 string21 string22 string31 string32)
+        expect( reading.watch.watched_files_collection.paths ).to eq [ file_path, file_path2, file_path3 ]
+      end
+    end
+
     context "when watching a directory with files using exit_after_read" do
       let(:opts) { super.merge(:exit_after_read => true, :max_open_files => 2) }
       let(:file_path3) { ::File.join(directory, "3.log") }
@@ -159,40 +206,45 @@ module FileWatch
       let(:listener6) { observer.listener_for(file_path6) }
 
       it "the file is read" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines).to eq(["line1", "line2"])
       end
+
       it "multiple files are read" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4"])
       end
+
       it "multiple files are read even if max_open_files is smaller then number of files" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         File.open(file_path4, "w") { |file| file.write("line3\nline4\n") }
         File.open(file_path5, "w") { |file| file.write("line5\nline6\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.lines.sort).to eq(["line1", "line2", "line3", "line4", "line5", "line6"])
       end
+
       it "file as marked as reading_completed" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
       end
+
       it "sincedb works correctly" do
-        File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
+        File.open(file_path3, "w") { |file| file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
         reading.subscribe(observer)
         sincedb_record_fields = File.read(sincedb_path).split(" ")
         position_field_index = 3
         expect(sincedb_record_fields[position_field_index]).to eq("12")
       end
+
       it "does not include new files added after start" do
         File.open(file_path3, "w") { |file|  file.write("line1\nline2\n") }
         reading.watch_this(watch_dir)
@@ -201,7 +253,6 @@ module FileWatch
         expect(listener3.lines).to eq(["line1", "line2"])
         expect(listener3.calls).to eq([:open, :accept, :accept, :eof, :delete, :reading_completed])
         expect(listener6.calls).to eq([])
-
       end
     
     end

data/spec/filewatch/settings_spec.rb

@@ -1,3 +1,6 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/inputs/friendly_durations'
+
 describe FileWatch::Settings do
 
   context "when create from options" do

data/spec/filewatch/spec_helper.rb

@@ -117,17 +117,12 @@ module FileWatch
     class Listener
       attr_reader :path, :lines, :calls
 
-      def initialize(path)
+      def initialize(path, lines)
         @path = path
-        @lines = Concurrent::Array.new
+        @lines = lines || Concurrent::Array.new
         @calls = Concurrent::Array.new
       end
 
-      def add_lines(lines)
-        @lines = lines
-        self
-      end
-
       def accept(line)
         @lines << line
         @calls << :accept
@@ -161,12 +156,7 @@ module FileWatch
     attr_reader :listeners
 
     def initialize(combined_lines = nil)
-      listener_proc = if combined_lines.nil?
-        lambda{|k| Listener.new(k) }
-      else
-        lambda{|k| Listener.new(k).add_lines(combined_lines) }
-      end
-      @listeners = Concurrent::Hash.new {|hash, key| hash[key] = listener_proc.call(key) }
+      @listeners = Concurrent::Hash.new { |hash, key| hash[key] = new_listener(key, combined_lines) }
     end
 
     def listener_for(path)
@@ -174,6 +164,14 @@ module FileWatch
     end
 
     def clear
-      @listeners.clear; end
+      @listeners.clear
+    end
+
+    private
+
+    def new_listener(path, lines = nil)
+      Listener.new(path, lines)
+    end
+
   end
 end

data/spec/filewatch/tailing_spec.rb

@@ -10,15 +10,19 @@ module FileWatch
     let(:file_path)  { ::File.join(directory, "1#{suffix}.log") }
     let(:file_path2) { ::File.join(directory, "2#{suffix}.log") }
     let(:file_path3) { ::File.join(directory, "3#{suffix}.log") }
-    let(:max)   { 4095 }
+    let(:max) { 4095 }
     let(:stat_interval) { 0.1 }
     let(:discover_interval) { 4 }
     let(:start_new_files_at) { :end }
     let(:sincedb_path) { ::File.join(directory, "tailing.sdb") }
     let(:opts) do
       {
-        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_open_files => max,
-        :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path,
+        :stat_interval => stat_interval,
+        :start_new_files_at => start_new_files_at,
+        :max_open_files => max,
+        :delimiter => "\n",
+        :discover_interval => discover_interval,
+        :sincedb_path => sincedb_path,
         :file_sort_by => "path"
       }
     end
@@ -30,12 +34,11 @@ module FileWatch
 
     before do
       directory
-      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      wait(1.0).for { Dir.exist?(directory) }.to eq(true)
     end
 
     after do
       FileUtils.rm_rf(directory)
-      wait(1.0).for{Dir.exist?(directory)}.to eq(false)
     end
 
     describe "max open files (set to 1)" do
@@ -95,16 +98,16 @@ module FileWatch
       let(:actions) do
         RSpec::Sequencing
           .run("create file") do
-            File.open(file_path, "wb") { |file|  file.write("lineA\nlineB\n") }
+            File.open(file_path, "wb") { |file| file.write("lineA\nlineB\n") }
           end
           .then_after(0.1, "begin watching") do
             tailing.watch_this(watch_dir)
           end
-          .then_after(2, "add content") do
-            File.open(file_path, "ab") { |file|  file.write("line1\nline2\n") }
+          .then_after(1.0, "add content") do
+            File.open(file_path, "ab") { |file| file.write("line1\nline2\n") }
           end
           .then("wait") do
-            wait(0.75).for{listener1.lines}.to eq(["line1", "line2"])
+            wait(0.75).for { listener1.lines }.to_not be_empty
           end
           .then("quit") do
             tailing.quit
@@ -113,7 +116,6 @@ module FileWatch
 
       it "only the new content is read" do
         actions.activate_quietly
-        tailing.watch_this(watch_dir)
         tailing.subscribe(observer)
         actions.assert_no_errors
         expect(listener1.calls).to eq([:open, :accept, :accept])
@@ -132,7 +134,7 @@ module FileWatch
             File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
           end
           .then("wait") do
-            wait(0.75).for{listener1.lines.size}.to eq(2)
+            wait(0.75).for { listener1.lines }.to_not be_empty
           end
           .then("quit") do
             tailing.quit
@@ -154,7 +156,7 @@ module FileWatch
       # so when a stat is taken on the file an error is raised
       let(:suffix) { "E" }
       let(:quit_after) { 0.2 }
-      let(:stat)  { double("stat", :size => 100, :modified_at => Time.now.to_f, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) }
+      let(:stat)  { double("stat", :size => 100, :modified_at => Time.now.to_f, :inode => 234567, :inode_struct => InodeStruct.new("234567", 1, 5)) }
       let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) }
       before do
         allow(stat).to receive(:restat).and_raise(Errno::ENOENT)

data/spec/filewatch/watched_file_spec.rb

@@ -35,5 +35,35 @@ module FileWatch
         expect(watched_file.recent_states).to eq([:watched, :active, :watched, :closed, :watched, :active, :unwatched, :active])
       end
     end
+
+    context 'restat' do
+
+      let(:directory) { Stud::Temporary.directory }
+      let(:file_path) { ::File.join(directory, "restat.file.txt") }
+      let(:pathname) { Pathname.new(file_path) }
+
+      before { FileUtils.touch file_path, :mtime => Time.now - 300 }
+
+      it 'reports false value when no changes' do
+        file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new)
+        mtime = file.modified_at
+        expect( file.modified_at_changed? ).to be false
+        expect( file.restat! ).to be_falsy
+        expect( file.modified_at_changed? ).to be false
+        expect( file.modified_at ).to eql mtime
+        expect( file.modified_at(true) ).to eql mtime
+      end
+
+      it 'reports truthy when changes detected' do
+        file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new)
+        mtime = file.modified_at
+        expect( file.modified_at_changed? ).to be false
+        FileUtils.touch file_path
+        expect( file.restat! ).to be_truthy
+        expect( file.modified_at_changed? ).to be true
+        expect( file.modified_at ).to eql mtime # until updated
+        expect( file.modified_at(true) ).to be > mtime
+      end
+    end
   end
 end

data/spec/filewatch/watched_files_collection_spec.rb

@@ -4,15 +4,18 @@ require_relative 'spec_helper'
 module FileWatch
   describe WatchedFilesCollection do
     let(:time) { Time.now }
-    let(:filepath1){"/var/log/z.log"}
-    let(:filepath2){"/var/log/m.log"}
-    let(:filepath3){"/var/log/a.log"}
-    let(:stat1)  { double("stat1", :size => 98, :modified_at => time - 30, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
-    let(:stat2)  { double("stat2", :size => 99, :modified_at => time - 20, :identifier => nil, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
-    let(:stat3)  { double("stat3", :size => 100, :modified_at => time, :identifier => nil, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) }
+    let(:filepath1) { "/var/log/z.log" }
+    let(:filepath2) { "/var/log/m.log" }
+    let(:filepath3) { "/var/log/a.log" }
+    let(:filepath4) { "/var/log/b.log" }
+    let(:stat1) { double("stat1", :size => 98, :modified_at => time - 30, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
+    let(:stat2) { double("stat2", :size => 99, :modified_at => time - 20, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
+    let(:stat3) { double("stat3", :size => 100, :modified_at => time, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) }
+    let(:stat4) { double("stat4", :size => 99, :modified_at => time, :inode => 234570, :inode_struct => InodeStruct.new("234570", 3, 2)) }
     let(:wf1) { WatchedFile.new(filepath1, stat1, Settings.new) }
     let(:wf2) { WatchedFile.new(filepath2, stat2, Settings.new) }
     let(:wf3) { WatchedFile.new(filepath3, stat3, Settings.new) }
+    let(:wf4) { WatchedFile.new(filepath4, stat4, Settings.new) }
 
     context "sort by last_modified in ascending order" do
       let(:sort_by) { "last_modified" }
@@ -20,12 +23,29 @@ module FileWatch
 
       it "sorts earliest modified first" do
         collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        expect(collection.empty?).to be true
         collection.add(wf2)
+        expect(collection.empty?).to be false
         expect(collection.values).to eq([wf2])
         collection.add(wf3)
         expect(collection.values).to eq([wf2, wf3])
         collection.add(wf1)
         expect(collection.values).to eq([wf1, wf2, wf3])
+        expect(collection.keys.size).to eq 3
+      end
+
+      it "sorts by path when mtime is same" do
+        collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        expect(collection.size).to eq 0
+        collection.add(wf2)
+        collection.add(wf4)
+        collection.add(wf1)
+        expect(collection.size).to eq 3
+        expect(collection.values).to eq([wf1, wf2, wf4])
+        collection.add(wf3)
+        expect(collection.size).to eq 4
+        expect(collection.values).to eq([wf1, wf2, wf3, wf4])
+        expect(collection.keys.size).to eq 4
       end
     end
 
@@ -74,7 +94,7 @@ module FileWatch
       end
     end
 
-    context "when delete called" do
+    context "remove_paths" do
       let(:sort_by) { "path" }
       let(:sort_direction) { "desc" }
 
@@ -85,9 +105,43 @@ module FileWatch
         collection.add(wf3)
         expect(collection.keys).to eq([filepath1, filepath2, filepath3])
 
-        collection.remove_paths([filepath2,filepath3])
+        ret = collection.remove_paths([filepath2, filepath3])
+        expect(ret).to eq 2
         expect(collection.keys).to eq([filepath1])
         expect(collection.values.size).to eq 1
+
+        ret = collection.remove_paths([filepath2])
+        expect(ret).to eq 0
+      end
+    end
+
+    context "update" do
+      let(:sort_by) { "last_modified" }
+      let(:sort_direction) { "asc" }
+
+      let(:re_stat1) { double("restat1", :size => 99, :modified_at => time, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
+      let(:re_stat2) { double("restat2", :size => 99, :modified_at => time, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
+
+      it "updates entry with changed mtime" do
+        collection = described_class.new(Settings.from_options(:file_sort_by => sort_by, :file_sort_direction => sort_direction))
+        collection.add(wf1)
+        collection.add(wf2)
+        collection.add(wf3)
+        expect(collection.files).to eq([wf1, wf2, wf3])
+
+        wf2.send(:set_stat, re_stat2)
+        expect( wf2.modified_at_changed? ).to be_truthy
+
+        collection.update wf2
+        expect(collection.files).to eq([wf1, wf3, wf2])
+
+        wf1.send(:set_stat, re_stat1)
+        expect( wf1.modified_at_changed? ).to be_truthy
+        collection.update wf1
+        expect(collection.files).to eq([wf3, wf2, wf1])
+
+        collection.add(wf4)
+        expect(collection.files).to eq([wf3, wf4, wf2, wf1])
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 4.1.18
+  version: 4.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-30 00:00:00.000000000 Z
+date: 2020-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,6 +58,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -178,6 +192,7 @@ files:
 - lib/filewatch/observing_base.rb
 - lib/filewatch/observing_read.rb
 - lib/filewatch/observing_tail.rb
+- lib/filewatch/processor.rb
 - lib/filewatch/read_mode/handlers/base.rb
 - lib/filewatch/read_mode/handlers/read_file.rb
 - lib/filewatch/read_mode/handlers/read_zip_file.rb