logstash-input-file 4.0.5 → 4.1.0

data/logstash-input-file.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '4.0.5'
-  s.licenses        = ['Apache License (2.0)']
+  s.version         = '4.1.0'
+  s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "JAR_VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -24,11 +24,12 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'addressable'
-  s.add_runtime_dependency 'filewatch', ['>= 0.8.1', '~> 0.8']
   s.add_runtime_dependency 'logstash-codec-multiline', ['~> 3.0']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'rspec-sequencing'
+  s.add_development_dependency "rspec-wait"
+  s.add_development_dependency 'timecop'
 end

data/spec/filewatch/buftok_spec.rb

@@ -0,0 +1,24 @@
+require_relative 'spec_helper'
+
+describe FileWatch::BufferedTokenizer do
+
+  context "when using the default delimiter" do
+    it "splits the lines correctly" do
+      expect(subject.extract("hello\nworld\n")).to eq ["hello", "world"]
+    end
+
+    it "holds partial lines back until a token is found" do
+      buffer = described_class.new
+      expect(buffer.extract("hello\nwor")).to eq ["hello"]
+      expect(buffer.extract("ld\n")).to eq ["world"]
+    end
+  end
+
+  context "when passing a custom delimiter" do
+    subject { FileWatch::BufferedTokenizer.new("\r\n") }
+
+    it "splits the lines correctly" do
+      expect(subject.extract("hello\r\nworld\r\n")).to eq ["hello", "world"]
+    end
+  end
+end

data/spec/filewatch/reading_spec.rb

@@ -0,0 +1,128 @@
+
+require 'stud/temporary'
+require_relative 'spec_helper'
+require 'filewatch/observing_read'
+
+module FileWatch
+  describe Watch do
+    before(:all) do
+      @thread_abort = Thread.abort_on_exception
+      Thread.abort_on_exception = true
+    end
+
+    after(:all) do
+      Thread.abort_on_exception = @thread_abort
+    end
+
+    let(:directory) { Stud::Temporary.directory }
+    let(:watch_dir) { ::File.join(directory, "*.log") }
+    let(:file_path) { ::File.join(directory, "1.log") }
+    let(:sincedb_path) { ::File.join(Stud::Temporary.directory, "reading.sdb") }
+    let(:stat_interval) { 0.1 }
+    let(:discover_interval) { 4 }
+    let(:start_new_files_at) { :end } # should be irrelevant for read mode
+    let(:opts) do
+      {
+        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at,
+        :delimiter => "\n", :discover_interval => discover_interval,
+        :ignore_older => 3600, :sincedb_path => sincedb_path
+      }
+    end
+    let(:observer) { TestObserver.new }
+    let(:reading) { ObservingRead.new(opts) }
+    let(:actions) do
+      RSpec::Sequencing.run_after(0.45, "quit after a short time") do
+        reading.quit
+      end
+    end
+
+    after do
+      FileUtils.rm_rf(directory) unless directory =~ /fixture/
+    end
+
+    context "when watching a directory with files" do
+      let(:directory) { Stud::Temporary.directory }
+      let(:watch_dir) { ::File.join(directory, "*.log") }
+      let(:file_path) { ::File.join(directory, "1.log") }
+
+      it "the file is read" do
+        File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+        actions.activate
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
+      end
+    end
+
+    context "when watching a directory with files using striped reading" do
+      let(:directory) { Stud::Temporary.directory }
+      let(:watch_dir) { ::File.join(directory, "*.log") }
+      let(:file_path1) { ::File.join(directory, "1.log") }
+      let(:file_path2) { ::File.join(directory, "2.log") }
+      # use a chunk size that does not align with the line boundaries
+      let(:opts) { super.merge(:file_chunk_size => 10, :file_chunk_count => 1)}
+      let(:lines) { [] }
+      let(:observer) { TestObserver.new(lines) }
+
+      it "the files are read seemingly in parallel" do
+        File.open(file_path1, "w") { |file|  file.write("string1\nstring2\n") }
+        File.open(file_path2, "w") { |file|  file.write("stringA\nstringB\n") }
+        actions.activate
+        reading.watch_this(watch_dir)
+        reading.subscribe(observer)
+        if lines.first == "stringA"
+          expect(lines).to eq(%w(stringA string1 stringB string2))
+        else
+          expect(lines).to eq(%w(string1 stringA string2 stringB))
+        end
+      end
+    end
+
+    describe "reading fixtures" do
+      let(:directory) { FIXTURE_DIR }
+
+      context "for an uncompressed file" do
+        let(:watch_dir) { ::File.join(directory, "unc*.log") }
+        let(:file_path) { ::File.join(directory, 'uncompressed.log') }
+
+        it "the file is read" do
+          FileWatch.make_fixture_current(file_path)
+          actions.activate
+          reading.watch_this(watch_dir)
+          reading.subscribe(observer)
+          expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
+          expect(observer.listener_for(file_path).lines.size).to eq(2)
+        end
+      end
+
+      context "for another uncompressed file" do
+        let(:watch_dir) { ::File.join(directory, "invalid*.log") }
+        let(:file_path) { ::File.join(directory, 'invalid_utf8.gbk.log') }
+
+        it "the file is read" do
+          FileWatch.make_fixture_current(file_path)
+          actions.activate
+          reading.watch_this(watch_dir)
+          reading.subscribe(observer)
+          expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
+          expect(observer.listener_for(file_path).lines.size).to eq(2)
+        end
+      end
+
+      context "for a compressed file" do
+        let(:watch_dir) { ::File.join(directory, "compressed.*.gz") }
+        let(:file_path) { ::File.join(directory, 'compressed.log.gz') }
+
+        it "the file is read" do
+          FileWatch.make_fixture_current(file_path)
+          actions.activate
+          reading.watch_this(watch_dir)
+          reading.subscribe(observer)
+          expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :eof, :delete])
+          expect(observer.listener_for(file_path).lines.size).to eq(2)
+        end
+      end
+    end
+  end
+end

data/spec/filewatch/sincedb_record_serializer_spec.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+require_relative 'spec_helper'
+require 'filewatch/settings'
+require 'filewatch/sincedb_record_serializer'
+
+module FileWatch
+  describe SincedbRecordSerializer do
+    let(:opts) { Hash.new }
+    let(:io) { StringIO.new }
+    let(:db) { Hash.new }
+
+    subject { described_class.new(Settings.days_to_seconds(14)) }
+
+    context "deserialize from IO" do
+      it 'reads V1 records' do
+        io.write("5391299 1 4 12\n")
+        subject.deserialize(io) do |inode_struct, sincedb_value|
+          expect(inode_struct.inode).to eq("5391299")
+          expect(inode_struct.maj).to eq(1)
+          expect(inode_struct.min).to eq(4)
+          expect(sincedb_value.position).to eq(12)
+        end
+      end
+
+      it 'reads V2 records from an IO object' do
+        now = Time.now.to_f
+        io.write("5391299 1 4 12 #{now} /a/path/to/1.log\n")
+        subject.deserialize(io) do |inode_struct, sincedb_value|
+          expect(inode_struct.inode).to eq("5391299")
+          expect(inode_struct.maj).to eq(1)
+          expect(inode_struct.min).to eq(4)
+          expect(sincedb_value.position).to eq(12)
+          expect(sincedb_value.last_changed_at).to eq(now)
+          expect(sincedb_value.path_in_sincedb).to eq("/a/path/to/1.log")
+        end
+      end
+    end
+
+    context "serialize to IO" do
+      it "writes db entries" do
+        now = Time.now.to_f
+        inode_struct = InodeStruct.new("42424242", 2, 5)
+        sincedb_value = SincedbValue.new(42, now)
+        db[inode_struct] = sincedb_value
+        subject.serialize(db, io)
+        expect(io.string).to eq("42424242 2 5 42 #{now}\n")
+      end
+
+      it "does not write expired db entries to an IO object" do
+        twelve_days_ago = Time.now.to_f - (12.0*24*3600)
+        sixteen_days_ago = twelve_days_ago - (4.0*24*3600)
+        db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, twelve_days_ago)
+        db[InodeStruct.new("18181818", 1, 6)] = SincedbValue.new(99, sixteen_days_ago)
+        subject.serialize(db, io)
+        expect(io.string).to eq("42424242 2 5 42 #{twelve_days_ago}\n")
+      end
+    end
+
+    context "given a non default `sincedb_clean_after`" do
+      it "does not write expired db entries to an IO object" do
+        subject.update_sincedb_value_expiry_from_days(2)
+        one_day_ago = Time.now.to_f - (1.0*24*3600)
+        three_days_ago = one_day_ago - (2.0*24*3600)
+        db[InodeStruct.new("42424242", 2, 5)] = SincedbValue.new(42, one_day_ago)
+        db[InodeStruct.new("18181818", 1, 6)] = SincedbValue.new(99, three_days_ago)
+        subject.serialize(db, io)
+        expect(io.string).to eq("42424242 2 5 42 #{one_day_ago}\n")
+      end
+    end
+  end
+end

data/spec/filewatch/spec_helper.rb

@@ -0,0 +1,120 @@
+require "rspec_sequencing"
+require 'rspec/wait'
+require "logstash/devutils/rspec/spec_helper"
+require "timecop"
+
+def formatted_puts(text)
+  cfg = RSpec.configuration
+  return unless cfg.formatters.first.is_a?(
+        RSpec::Core::Formatters::DocumentationFormatter)
+  txt = cfg.format_docstrings_block.call(text)
+  cfg.output_stream.puts "    #{txt}"
+end
+
+unless RSpec::Matchers.method_defined?(:receive_call_and_args)
+  RSpec::Matchers.define(:receive_call_and_args) do |m, args|
+    match do |actual|
+      actual.trace_for(m) == args
+     end
+
+    failure_message do
+      "Expecting method #{m} to receive: #{args} but got: #{actual.trace_for(m)}"
+    end
+  end
+end
+
+require 'filewatch/bootstrap'
+
+module FileWatch
+
+  FIXTURE_DIR = File.join('spec', 'fixtures')
+
+  def self.make_file_older(path, seconds)
+    time = Time.now.to_f - seconds
+    ::File.utime(time, time, path)
+  end
+
+  def self.make_fixture_current(path, time = Time.now)
+    ::File.utime(time, time, path)
+  end
+
+  class TracerBase
+    def initialize
+      @tracer = []
+    end
+
+    def trace_for(symbol)
+      params = @tracer.map {|k,v| k == symbol ? v : nil}.compact
+      params.empty? ? false : params
+    end
+
+    def clear
+      @tracer.clear
+    end
+  end
+
+  module NullCallable
+    def self.call
+    end
+  end
+
+  class TestObserver
+    class Listener
+      attr_reader :path, :lines, :calls
+
+      def initialize(path)
+        @path = path
+        @lines = []
+        @calls = []
+      end
+
+      def add_lines(lines)
+        @lines = lines
+        self
+      end
+
+      def accept(line)
+        @lines << line
+        @calls << :accept
+      end
+
+      def deleted
+        @calls << :delete
+      end
+
+      def opened
+        @calls << :open
+      end
+
+      def error
+        @calls << :error
+      end
+
+      def eof
+        @calls << :eof
+      end
+
+      def timed_out
+        @calls << :timed_out
+      end
+    end
+
+    attr_reader :listeners
+
+    def initialize(combined_lines = nil)
+      listener_proc = if combined_lines.nil?
+        lambda{|k| Listener.new(k) }
+      else
+        lambda{|k| Listener.new(k).add_lines(combined_lines) }
+      end
+      @listeners = Hash.new {|hash, key| hash[key] = listener_proc.call(key) }
+    end
+
+    def listener_for(path)
+      @listeners[path]
+    end
+
+    def clear
+      @listeners.clear; end
+  end
+end

data/spec/filewatch/tailing_spec.rb

@@ -0,0 +1,440 @@
+
+require 'stud/temporary'
+require_relative 'spec_helper'
+require 'filewatch/observing_tail'
+
+LogStash::Logging::Logger::configure_logging("WARN")
+# LogStash::Logging::Logger::configure_logging("DEBUG")
+
+module FileWatch
+  describe Watch do
+    before(:all) do
+      @thread_abort = Thread.abort_on_exception
+      Thread.abort_on_exception = true
+    end
+
+    after(:all) do
+      Thread.abort_on_exception = @thread_abort
+    end
+
+    let(:directory) { Stud::Temporary.directory }
+    let(:watch_dir) { ::File.join(directory, "*.log") }
+    let(:file_path) { ::File.join(directory, "1.log") }
+    let(:max)   { 4095 }
+    let(:stat_interval) { 0.1 }
+    let(:discover_interval) { 4 }
+    let(:start_new_files_at) { :beginning }
+    let(:sincedb_path) { ::File.join(directory, "tailing.sdb") }
+    let(:opts) do
+      {
+        :stat_interval => stat_interval, :start_new_files_at => start_new_files_at, :max_active => max,
+        :delimiter => "\n", :discover_interval => discover_interval, :sincedb_path => sincedb_path
+      }
+    end
+    let(:observer) { TestObserver.new }
+    let(:tailing) { ObservingTail.new(opts) }
+
+    after do
+      FileUtils.rm_rf(directory)
+    end
+
+    describe "max open files (set to 1)" do
+      let(:max) { 1 }
+      let(:file_path2) { File.join(directory, "2.log") }
+      let(:wait_before_quit) { 0.15 }
+      let(:stat_interval) { 0.01 }
+      let(:discover_interval) { 4 }
+      let(:actions) do
+        RSpec::Sequencing
+          .run_after(wait_before_quit, "quit after a short time") do
+            tailing.quit
+          end
+      end
+
+      before do
+        ENV["FILEWATCH_MAX_FILES_WARN_INTERVAL"] = "0"
+        File.open(file_path, "wb")  { |file| file.write("line1\nline2\n") }
+        File.open(file_path2, "wb") { |file| file.write("lineA\nlineB\n") }
+      end
+
+      context "when max_active is 1" do
+
+        it "without close_older set, opens only 1 file" do
+          actions.activate
+          tailing.watch_this(watch_dir)
+          tailing.subscribe(observer)
+          expect(tailing.settings.max_active).to eq(max)
+          file1_calls = observer.listener_for(file_path).calls
+          file2_calls = observer.listener_for(file_path2).calls
+          # file glob order is OS dependent
+          if file1_calls.empty?
+            expect(observer.listener_for(file_path2).lines).to eq(["lineA", "lineB"])
+            expect(file2_calls).to eq([:open, :accept, :accept])
+          else
+            expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
+            expect(file1_calls).to eq([:open, :accept, :accept])
+            expect(file2_calls).to be_empty
+          end
+        end
+      end
+
+      context "when close_older is set" do
+        let(:wait_before_quit) { 0.8 }
+        let(:opts) { super.merge(:close_older => 0.2, :max_active => 1, :stat_interval => 0.1) }
+        it "opens both files" do
+          actions.activate
+          tailing.watch_this(watch_dir)
+          tailing.subscribe(observer)
+          expect(tailing.settings.max_active).to eq(1)
+          filelistener_1 = observer.listener_for(file_path)
+          filelistener_2 = observer.listener_for(file_path2)
+          expect(filelistener_2.calls).to eq([:open, :accept, :accept, :timed_out])
+          expect(filelistener_2.lines).to eq(["lineA", "lineB"])
+          expect(filelistener_1.calls).to eq([:open, :accept, :accept, :timed_out])
+          expect(filelistener_1.lines).to eq(["line1", "line2"])
+        end
+      end
+    end
+
+    context "when watching a directory with files" do
+      let(:start_new_files_at) { :beginning }
+      let(:actions) do
+        RSpec::Sequencing.run_after(0.45, "quit after a short time") do
+          tailing.quit
+        end
+      end
+
+      it "the file is read" do
+        File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+        actions.activate
+        tailing.watch_this(watch_dir)
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
+      end
+    end
+
+    context "when watching a directory without files and one is added" do
+      let(:start_new_files_at) { :beginning }
+      before do
+        tailing.watch_this(watch_dir)
+        RSpec::Sequencing
+          .run_after(0.25, "create file") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+          end
+          .then_after(0.45, "quit after a short time") do
+            tailing.quit
+          end
+      end
+
+      it "the file is read" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
+      end
+    end
+
+    describe "given a previously discovered file" do
+      # these tests rely on the fact that the 'filepath' does not exist on disk
+      # it simulates that the user deleted the file
+      # so when a stat is taken on the file an error is raised
+      let(:quit_after) { 0.2 }
+      let(:stat)  { double("stat", :size => 100, :ctime => Time.now, :mtime => Time.now, :ino => 234567, :dev_major => 3, :dev_minor => 2) }
+      let(:watched_file) { WatchedFile.new(file_path, stat, tailing.settings) }
+
+      before do
+        tailing.watch.watched_files_collection.add(watched_file)
+        watched_file.initial_completed
+      end
+
+      context "when a close operation occurs" do
+        before { watched_file.close }
+        it "is removed from the watched_files_collection" do
+          expect(tailing.watch.watched_files_collection).not_to be_empty
+          RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
+          tailing.subscribe(observer)
+          expect(tailing.watch.watched_files_collection).to be_empty
+          expect(observer.listener_for(file_path).calls).to eq([:delete])
+        end
+      end
+
+      context "an ignore operation occurs" do
+        before { watched_file.ignore }
+        it "is removed from the watched_files_collection" do
+          RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
+          tailing.subscribe(observer)
+          expect(tailing.watch.watched_files_collection).to be_empty
+          expect(observer.listener_for(file_path).calls).to eq([:delete])
+        end
+      end
+
+      context "when subscribed and a watched file is no longer readable" do
+        before { watched_file.watch }
+        it "is removed from the watched_files_collection" do
+          RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
+          tailing.subscribe(observer)
+          expect(tailing.watch.watched_files_collection).to be_empty
+          expect(observer.listener_for(file_path).calls).to eq([:delete])
+        end
+      end
+
+      context "when subscribed and an active file is no longer readable" do
+        before { watched_file.activate }
+        it "is removed from the watched_files_collection" do
+          RSpec::Sequencing.run_after(quit_after, "quit") { tailing.quit }
+          tailing.subscribe(observer)
+          expect(tailing.watch.watched_files_collection).to be_empty
+          expect(observer.listener_for(file_path).calls).to eq([:delete])
+        end
+      end
+    end
+
+    context "when a processed file shrinks" do
+      let(:discover_interval) { 100 }
+      before do
+        RSpec::Sequencing
+        .run("create file") do
+          File.open(file_path, "wb") { |file|  file.write("line1\nline2\nline3\nline4\n") }
+        end
+        .then_after(0.25, "start watching after files are written") do
+          tailing.watch_this(watch_dir)
+        end
+        .then_after(0.25, "truncate file and write new content") do
+          File.truncate(file_path, 0)
+          File.open(file_path, "wb") { |file|  file.write("lineA\nlineB\n") }
+        end
+        .then_after(0.25, "quit after a short time") do
+          tailing.quit
+        end
+      end
+
+      it "new changes to the shrunk file are read from the beginning" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept, :accept, :accept])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4", "lineA", "lineB"])
+      end
+    end
+
+    context "when watching a directory with files and a file is renamed to not match glob" do
+      let(:new_file_path) { file_path + ".old" }
+      before do
+        RSpec::Sequencing
+          .run("create file") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+          end
+          .then_after(0.25, "start watching after files are written") do
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(0.55, "rename file") do
+            FileUtils.mv(file_path, new_file_path)
+          end
+          .then_after(0.55, "then write to renamed file") do
+            File.open(new_file_path, "ab") { |file|  file.write("line3\nline4\n") }
+          end
+          .then_after(0.45, "quit after a short time") do
+            tailing.quit
+          end
+      end
+
+      it "changes to the renamed file are not read" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :delete])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
+        expect(observer.listener_for(new_file_path).calls).to eq([])
+        expect(observer.listener_for(new_file_path).lines).to eq([])
+      end
+    end
+
+    context "when watching a directory with files and a file is renamed to match glob" do
+      let(:new_file_path) { file_path + "2.log" }
+      let(:opts) { super.merge(:close_older => 0) }
+      before do
+        RSpec::Sequencing
+          .run("create file") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+          end
+          .then_after(0.15, "start watching after files are written") do
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(0.25, "rename file") do
+            FileUtils.mv(file_path, new_file_path)
+          end
+          .then("then write to renamed file") do
+            File.open(new_file_path, "ab") { |file|  file.write("line3\nline4\n") }
+          end
+          .then_after(0.55, "quit after a short time") do
+            tailing.quit
+          end
+      end
+
+      it "the first set of lines are not re-read" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :delete])
+        expect(observer.listener_for(new_file_path).lines).to eq(["line3", "line4"])
+        expect(observer.listener_for(new_file_path).calls).to eq([:open, :accept, :accept, :timed_out])
+      end
+    end
+
+    context "when watching a directory with files and data is appended" do
+      before do
+        RSpec::Sequencing
+          .run("create file") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+          end
+          .then_after(0.25, "start watching after file is written") do
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(0.45, "append more lines to the file") do
+            File.open(file_path, "ab") { |file|  file.write("line3\nline4\n") }
+          end
+          .then_after(0.45, "quit after a short time") do
+            tailing.quit
+          end
+      end
+
+      it "appended lines are read after an EOF" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :accept, :accept])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"])
+      end
+    end
+
+    context "when close older expiry is enabled" do
+      let(:opts) { super.merge(:close_older => 1) }
+      before do
+        RSpec::Sequencing
+          .run("create file") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+          end
+          .then("start watching before file ages more than close_older") do
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(2.1, "quit after allowing time to close the file") do
+            tailing.quit
+          end
+      end
+
+      it "lines are read and the file times out" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
+      end
+    end
+
+    context "when close older expiry is enabled and after timeout the file is appended-to" do
+      let(:opts) { super.merge(:close_older => 1) }
+      before do
+        RSpec::Sequencing
+          .run("create file") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+          end
+          .then("start watching before file ages more than close_older") do
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(2.1, "append more lines to file after file ages more than close_older") do
+            File.open(file_path, "ab") { |file|  file.write("line3\nline4\n") }
+          end
+          .then_after(2.1, "quit after allowing time to close the file") do
+            tailing.quit
+          end
+      end
+
+      it "all lines are read" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out, :open, :accept, :accept, :timed_out])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2", "line3", "line4"])
+      end
+    end
+
+    context "when ignore older expiry is enabled and all files are already expired" do
+      let(:opts) { super.merge(:ignore_older => 1) }
+      before do
+        RSpec::Sequencing
+          .run("create file older than ignore_older and watch") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+            FileWatch.make_file_older(file_path, 15)
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(1.1, "quit") do
+            tailing.quit
+          end
+      end
+
+      it "no files are read" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([])
+        expect(observer.listener_for(file_path).lines).to eq([])
+      end
+    end
+
+    context "when ignore_older is less than close_older and all files are not expired" do
+      let(:opts) { super.merge(:ignore_older => 1, :close_older => 1.5) }
+      before do
+        RSpec::Sequencing
+          .run("create file") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+          end
+          .then("start watching before file age reaches ignore_older") do
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(1.75, "quit after allowing time to close the file") do
+            tailing.quit
+          end
+      end
+
+      it "reads lines normally" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
+        expect(observer.listener_for(file_path).lines).to eq(["line1", "line2"])
+      end
+    end
+
+    context "when ignore_older is less than close_older and all files are expired" do
+      let(:opts) { super.merge(:ignore_older => 10, :close_older => 1) }
+      before do
+        RSpec::Sequencing
+          .run("create file older than ignore_older and watch") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+            FileWatch.make_file_older(file_path, 15)
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(1.5, "quit after allowing time to check the files") do
+            tailing.quit
+          end
+      end
+
+      it "no files are read" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).calls).to eq([])
+        expect(observer.listener_for(file_path).lines).to eq([])
+      end
+    end
+
+    context "when ignore older and close older expiry is enabled and after timeout the file is appended-to" do
+      let(:opts) { super.merge(:ignore_older => 20, :close_older => 1) }
+      before do
+        RSpec::Sequencing
+          .run("create file older than ignore_older and watch") do
+            File.open(file_path, "wb") { |file|  file.write("line1\nline2\n") }
+            FileWatch.make_file_older(file_path, 25)
+            tailing.watch_this(watch_dir)
+          end
+          .then_after(0.15, "append more lines to file after file ages more than ignore_older") do
+            File.open(file_path, "ab") { |file|  file.write("line3\nline4\n") }
+          end
+          .then_after(1.25, "quit after allowing time to close the file") do
+            tailing.quit
+          end
+      end
+
+      it "reads the added lines only" do
+        tailing.subscribe(observer)
+        expect(observer.listener_for(file_path).lines).to eq(["line3", "line4"])
+        expect(observer.listener_for(file_path).calls).to eq([:open, :accept, :accept, :timed_out])
+      end
+    end
+  end
+end
+
+