logstash-input-file 4.1.3 → 4.1.4

data/spec/filewatch/watched_file_spec.rb

@@ -8,9 +8,9 @@ module FileWatch
 
     context 'Given two instances of the same file' do
       it 'their sincedb_keys should equate' do
-        wf_key1 = WatchedFile.new(pathname, pathname.stat, Settings.new).sincedb_key
+        wf_key1 = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new).sincedb_key
         hash_db = { wf_key1 => 42 }
-        wf_key2 = WatchedFile.new(pathname, pathname.stat, Settings.new).sincedb_key
+        wf_key2 = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new).sincedb_key
         expect(wf_key1).to eq(wf_key2)
         expect(wf_key1).to eql(wf_key2)
         expect(wf_key1.hash).to eq(wf_key2.hash)
@@ -20,7 +20,7 @@ module FileWatch
 
     context 'Given a barrage of state changes' do
       it 'only the previous N state changes are remembered' do
-        watched_file = WatchedFile.new(pathname, pathname.stat, Settings.new)
+        watched_file = WatchedFile.new(pathname, PathStatClass.new(pathname), Settings.new)
         watched_file.ignore
         watched_file.watch
         watched_file.activate

data/spec/filewatch/watched_files_collection_spec.rb

@@ -4,9 +4,9 @@ require_relative 'spec_helper'
 module FileWatch
   describe WatchedFilesCollection do
     let(:time) { Time.now }
-    let(:stat1)  { double("stat1", :size => 98, :ctime => time - 30, :mtime => time - 30, :ino => 234567, :dev_major => 3, :dev_minor => 2) }
-    let(:stat2)  { double("stat2", :size => 99, :ctime => time - 20, :mtime => time - 20, :ino => 234568, :dev_major => 3, :dev_minor => 2) }
-    let(:stat3)  { double("stat3", :size => 100, :ctime => time, :mtime => time, :ino => 234569, :dev_major => 3, :dev_minor => 2) }
+    let(:stat1)  { double("stat1", :size => 98, :modified_at => time - 30, :identifier => nil, :inode => 234567, :inode_struct => InodeStruct.new("234567", 3, 2)) }
+    let(:stat2)  { double("stat2", :size => 99, :modified_at => time - 20, :identifier => nil, :inode => 234568, :inode_struct => InodeStruct.new("234568", 3, 2)) }
+    let(:stat3)  { double("stat3", :size => 100, :modified_at => time, :identifier => nil, :inode => 234569, :inode_struct => InodeStruct.new("234569", 3, 2)) }
     let(:wf1) { WatchedFile.new("/var/log/z.log", stat1, Settings.new) }
     let(:wf2) { WatchedFile.new("/var/log/m.log", stat2, Settings.new) }
     let(:wf3) { WatchedFile.new("/var/log/a.log", stat3, Settings.new) }

data/spec/filewatch/winhelper_spec.rb

@@ -3,7 +3,7 @@ require "stud/temporary"
 require "fileutils"
 
 if Gem.win_platform?
-  require "lib/filewatch/winhelper"
+  require "filewatch/winhelper"
 
   describe Winhelper do
     let(:path) { Stud::Temporary.file.path }
@@ -13,11 +13,10 @@ if Gem.win_platform?
     end
 
     it "return a unique file identifier" do
-      volume_serial, file_index_low, file_index_high = Winhelper.GetWindowsUniqueFileIdentifier(path).split("").map(&:to_i)
+      identifier = Winhelper.identifier_from_path(path)
 
-      expect(volume_serial).not_to eq(0)
-      expect(file_index_low).not_to eq(0)
-      expect(file_index_high).not_to eq(0)
+      expect(identifier).not_to eq("unknown")
+      expect(identifier.count("-")).to eq(2)
     end
   end
 end

data/spec/helpers/logging_level_helper.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+
+ENV["LOG_AT"].tap do |level|
+  if !level.nil?
+    LogStash::Logging::Logger::configure_logging(level)
+    LOG_AT_HANDLED = true
+  end
+end

data/spec/helpers/rspec_wait_handler_helper.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+module RSpec
+  module Wait
+    module Handler
+      def handle_matcher(target, *args, &block)
+        # there is a similar patch in the rspec-wait repo since Nov, 19 2017
+        # it does not look like the author is interested in the change.
+        # - do not use Ruby Timeout
+        count = RSpec.configuration.wait_timeout.fdiv(RSpec.configuration.wait_delay).ceil
+        failure = nil
+        count.times do
+          begin
+            actual = target.respond_to?(:call) ? target.call : target
+            super(actual, *args, &block)
+            failure = nil
+          rescue RSpec::Expectations::ExpectationNotMetError => failure
+            sleep RSpec.configuration.wait_delay
+          end
+          break if failure.nil?
+        end
+        raise failure unless failure.nil?
+      end
+    end
+
+    # From: https://github.com/rspec/rspec-expectations/blob/v3.0.0/lib/rspec/expectations/handler.rb#L44-L63
+    class PositiveHandler < RSpec::Expectations::PositiveExpectationHandler
+      extend Handler
+    end
+
+    # From: https://github.com/rspec/rspec-expectations/blob/v3.0.0/lib/rspec/expectations/handler.rb#L66-L93
+    class NegativeHandler < RSpec::Expectations::NegativeExpectationHandler
+      extend Handler
+    end
+  end
+end
+
+RSPEC_WAIT_HANDLER_PATCHED = true

data/spec/helpers/spec_helper.rb

@@ -18,7 +18,7 @@ module FileInput
 
   class TracerBase
     def initialize
-      @tracer = []
+      @tracer = Concurrent::Array.new
     end
 
     def trace_for(symbol)
@@ -54,6 +54,9 @@ module FileInput
   end
 end
 
+require_relative "rspec_wait_handler_helper" unless defined? RSPEC_WAIT_HANDLER_PATCHED
+require_relative "logging_level_helper" unless defined? LOG_AT_HANDLED
+
 unless RSpec::Matchers.method_defined?(:receive_call_and_args)
   RSpec::Matchers.define(:receive_call_and_args) do |m, args|
     match do |actual|
@@ -66,3 +69,6 @@ unless RSpec::Matchers.method_defined?(:receive_call_and_args)
   end
 end
 
+ENV["LOG_AT"].tap do |level|
+  LogStash::Logging::Logger::configure_logging(level) unless level.nil?
+end

data/spec/inputs/file_read_spec.rb

@@ -9,21 +9,21 @@ require "tempfile"
 require "stud/temporary"
 require "logstash/codecs/multiline"
 
-FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n"
-
 describe LogStash::Inputs::File do
   describe "'read' mode testing with input(conf) do |pipeline, queue|" do
     it "should start at the beginning of an existing file and delete the file when done" do
-      tmpfile_path = Stud::Temporary.pathname
-      sincedb_path = Stud::Temporary.pathname
+      directory = Stud::Temporary.directory
+      tmpfile_path = ::File.join(directory, "A.log")
+      sincedb_path = ::File.join(directory, "readmode_A_sincedb.txt")
+      path_path = ::File.join(directory, "*.log")
 
       conf = <<-CONFIG
         input {
           file {
-            type => "blah"
-            path => "#{tmpfile_path}"
+            id => "blah"
+            path => "#{path_path}"
             sincedb_path => "#{sincedb_path}"
-            delimiter => "#{FILE_DELIMITER}"
+            delimiter => "|"
             mode => "read"
             file_completed_action => "delete"
           }
@@ -31,17 +31,49 @@ describe LogStash::Inputs::File do
       CONFIG
 
       File.open(tmpfile_path, "a") do |fd|
-        fd.puts("hello")
-        fd.puts("world")
+        fd.write("hello|world")
         fd.fsync
       end
 
       events = input(conf) do |pipeline, queue|
+        wait(0.5).for{File.exist?(tmpfile_path)}.to be_falsey
         2.times.collect { queue.pop }
       end
 
       expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world")
-      expect(File.exist?(tmpfile_path)).to be_falsey
+    end
+
+    it "should start at the beginning of an existing file and log the file when done" do
+      directory = Stud::Temporary.directory
+      tmpfile_path = ::File.join(directory, "A.log")
+      sincedb_path = ::File.join(directory, "readmode_A_sincedb.txt")
+      path_path = ::File.join(directory, "*.log")
+      log_completed_path = ::File.join(directory, "A_completed.txt")
+
+      conf = <<-CONFIG
+        input {
+          file {
+            id => "blah"
+            path => "#{path_path}"
+            sincedb_path => "#{sincedb_path}"
+            delimiter => "|"
+            mode => "read"
+            file_completed_action => "log"
+            file_completed_log_path => "#{log_completed_path}"
+          }
+        }
+      CONFIG
+
+      File.open(tmpfile_path, "a") do |fd|
+        fd.write("hello|world")
+        fd.fsync
+      end
+
+      events = input(conf) do |pipeline, queue|
+        wait(0.5).for{IO.read(log_completed_path)}.to match(/A\.log/)
+        2.times.collect { queue.pop }
+      end
+      expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world")
     end
   end
 
@@ -63,7 +95,6 @@ describe LogStash::Inputs::File do
             type => "blah"
             path => "#{tmpfile_path}"
             sincedb_path => "#{sincedb_path}"
-            delimiter => "#{FILE_DELIMITER}"
             mode => "read"
             file_completed_action => "log"
             file_completed_log_path => "#{log_completed_path}"
@@ -72,12 +103,12 @@ describe LogStash::Inputs::File do
         CONFIG
 
         events = input(conf) do |pipeline, queue|
+          wait(0.5).for{IO.read(log_completed_path)}.to match(/#{file_path.to_s}/)
           2.times.collect { queue.pop }
         end
 
         expect(events[0].get("message")).to start_with("2010-03-12   23:51")
         expect(events[1].get("message")).to start_with("2010-03-12   23:51")
-        expect(IO.read(log_completed_path)).to eq(file_path.to_s + "\n")
       end
 
     end
@@ -86,10 +117,11 @@ describe LogStash::Inputs::File do
       let(:file_path) { fixture_dir.join('uncompressed.log') }
 
       it "the file is read and the path is logged to the `file_completed_log_path` file" do
-        tmpfile_path = fixture_dir.join("unc*.log")
-        sincedb_path = Stud::Temporary.pathname
         FileInput.make_fixture_current(file_path.to_path)
-        log_completed_path = Stud::Temporary.pathname
+        tmpfile_path = fixture_dir.join("unc*.log")
+        directory = Stud::Temporary.directory
+        sincedb_path = ::File.join(directory, "readmode_B_sincedb.txt")
+        log_completed_path = ::File.join(directory, "B_completed.txt")
 
         conf = <<-CONFIG
         input {
@@ -97,7 +129,6 @@ describe LogStash::Inputs::File do
             type => "blah"
             path => "#{tmpfile_path}"
             sincedb_path => "#{sincedb_path}"
-            delimiter => "#{FILE_DELIMITER}"
             mode => "read"
             file_completed_action => "log"
             file_completed_log_path => "#{log_completed_path}"
@@ -106,23 +137,25 @@ describe LogStash::Inputs::File do
         CONFIG
 
         events = input(conf) do |pipeline, queue|
+          wait(0.5).for{IO.read(log_completed_path)}.to match(/uncompressed\.log/)
           2.times.collect { queue.pop }
         end
 
         expect(events[0].get("message")).to start_with("2010-03-12   23:51")
         expect(events[1].get("message")).to start_with("2010-03-12   23:51")
-        expect(IO.read(log_completed_path)).to eq(file_path.to_s + "\n")
       end
     end
 
     context "for a compressed file" do
       it "the file is read" do
-        tmpfile_path = fixture_dir.join("compressed.*.*")
-        sincedb_path = Stud::Temporary.pathname
         file_path = fixture_dir.join('compressed.log.gz')
         file_path2 = fixture_dir.join('compressed.log.gzip')
         FileInput.make_fixture_current(file_path.to_path)
-        log_completed_path = Stud::Temporary.pathname
+        FileInput.make_fixture_current(file_path2.to_path)
+        tmpfile_path = fixture_dir.join("compressed.*.*")
+        directory = Stud::Temporary.directory
+        sincedb_path = ::File.join(directory, "readmode_C_sincedb.txt")
+        log_completed_path = ::File.join(directory, "C_completed.txt")
 
         conf = <<-CONFIG
         input {
@@ -130,7 +163,6 @@ describe LogStash::Inputs::File do
             type => "blah"
             path => "#{tmpfile_path}"
             sincedb_path => "#{sincedb_path}"
-            delimiter => "#{FILE_DELIMITER}"
             mode => "read"
             file_completed_action => "log"
             file_completed_log_path => "#{log_completed_path}"
@@ -139,6 +171,7 @@ describe LogStash::Inputs::File do
         CONFIG
 
         events = input(conf) do |pipeline, queue|
+          wait(0.5).for{IO.read(log_completed_path).scan(/compressed\.log\.gz(ip)?/).size}.to eq(2)
           4.times.collect { queue.pop }
         end
 
@@ -146,9 +179,6 @@ describe LogStash::Inputs::File do
         expect(events[1].get("message")).to start_with("2010-03-12   23:51")
         expect(events[2].get("message")).to start_with("2010-03-12   23:51")
         expect(events[3].get("message")).to start_with("2010-03-12   23:51")
-        logged_completions = IO.read(log_completed_path).split
-        expect(logged_completions.first).to match(/compressed\.log\.(gzip|gz)$/)
-        expect(logged_completions.last).to match(/compressed\.log\.(gzip|gz)$/)
       end
     end
   end

data/spec/inputs/file_tail_spec.rb

@@ -9,7 +9,7 @@ require "logstash/codecs/multiline"
 
 # LogStash::Logging::Logger::configure_logging("DEBUG")
 
-TEST_FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n"
+TEST_FILE_DELIMITER = $/
 
 describe LogStash::Inputs::File do
   describe "'tail' mode testing with input(conf) do |pipeline, queue|" do
@@ -22,152 +22,159 @@ describe LogStash::Inputs::File do
       end
     end
 
-    it "should start at the beginning of an existing file" do
-      tmpfile_path = Stud::Temporary.pathname
-      sincedb_path = Stud::Temporary.pathname
-
-      conf = <<-CONFIG
-        input {
-          file {
-            type => "blah"
-            path => "#{tmpfile_path}"
-            start_position => "beginning"
-            sincedb_path => "#{sincedb_path}"
-            delimiter => "#{TEST_FILE_DELIMITER}"
+    let(:directory)    { Stud::Temporary.directory }
+    let(:sincedb_dir)  { Stud::Temporary.directory }
+    let(:tmpfile_path) { ::File.join(directory, "#{name}.txt") }
+    let(:sincedb_path) { ::File.join(sincedb_dir, "readmode_#{name}_sincedb.txt") }
+    let(:path_path)    { ::File.join(directory, "*.txt") }
+
+    context "for an existing file" do
+      let(:name) { "A" }
+      it "should start at the beginning" do
+        conf = <<-CONFIG
+          input {
+            file {
+              type => "blah"
+              path => "#{path_path}"
+              start_position => "beginning"
+              sincedb_path => "#{sincedb_path}"
+              delimiter => "#{TEST_FILE_DELIMITER}"
+            }
           }
-        }
-      CONFIG
+        CONFIG
 
-      File.open(tmpfile_path, "a") do |fd|
-        fd.puts("hello")
-        fd.puts("world")
-        fd.fsync
-      end
+        File.open(tmpfile_path, "a") do |fd|
+          fd.puts("hello")
+          fd.puts("world")
+          fd.fsync
+        end
 
-      events = input(conf) do |pipeline, queue|
-        2.times.collect { queue.pop }
+        events = input(conf) do |pipeline, queue|
+          2.times.collect { queue.pop }
+        end
+        expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world")
       end
-      expect(events.map{|e| e.get("message")}).to contain_exactly("hello", "world")
     end
 
-    it "should restart at the sincedb value" do
-      tmpfile_path = Stud::Temporary.pathname
-      sincedb_path = Stud::Temporary.pathname
-
-      conf = <<-CONFIG
-        input {
-          file {
-            type => "blah"
-            path => "#{tmpfile_path}"
-            start_position => "beginning"
-            sincedb_path => "#{sincedb_path}"
-            delimiter => "#{TEST_FILE_DELIMITER}"
+    context "running the input twice" do
+      let(:name) { "B" }
+      it "should restart at the sincedb value" do
+        conf = <<-CONFIG
+          input {
+            file {
+              type => "blah"
+              path => "#{path_path}"
+              start_position => "beginning"
+              sincedb_path => "#{sincedb_path}"
+              "file_sort_by" => "path"
+              delimiter => "#{TEST_FILE_DELIMITER}"
+            }
           }
-        }
-      CONFIG
+        CONFIG
 
-      File.open(tmpfile_path, "w") do |fd|
-        fd.puts("hello3")
-        fd.puts("world3")
-      end
+        File.open(tmpfile_path, "w") do |fd|
+          fd.puts("hello3")
+          fd.puts("world3")
+        end
 
-      events = input(conf) do |pipeline, queue|
-        2.times.collect { queue.pop }
-      end
+        events = input(conf) do |pipeline, queue|
+          2.times.collect { queue.pop }
+        end
 
-      expect(events.map{|e| e.get("message")}).to contain_exactly("hello3", "world3")
+        expect(events.map{|e| e.get("message")}).to contain_exactly("hello3", "world3")
 
-      File.open(tmpfile_path, "a") do |fd|
-        fd.puts("foo")
-        fd.puts("bar")
-        fd.puts("baz")
-        fd.fsync
-      end
+        File.open(tmpfile_path, "a") do |fd|
+          fd.puts("foo")
+          fd.puts("bar")
+          fd.puts("baz")
+          fd.fsync
+        end
 
-      events = input(conf) do |pipeline, queue|
-        3.times.collect { queue.pop }
+        events = input(conf) do |pipeline, queue|
+          3.times.collect { queue.pop }
+        end
+        messages = events.map{|e| e.get("message")}
+        expect(messages).to contain_exactly("foo", "bar", "baz")
       end
-      messages = events.map{|e| e.get("message")}
-      expect(messages).to contain_exactly("foo", "bar", "baz")
     end
 
-    it "should not overwrite existing path and host fields" do
-      tmpfile_path = Stud::Temporary.pathname
-      sincedb_path = Stud::Temporary.pathname
-
-      conf = <<-CONFIG
-        input {
-          file {
-            type => "blah"
-            path => "#{tmpfile_path}"
-            start_position => "beginning"
-            sincedb_path => "#{sincedb_path}"
-            delimiter => "#{TEST_FILE_DELIMITER}"
-            codec => "json"
+    context "when path and host fields exist" do
+      let(:name) { "C" }
+      it "should not overwrite them" do
+        conf = <<-CONFIG
+          input {
+            file {
+              type => "blah"
+              path => "#{path_path}"
+              start_position => "beginning"
+              sincedb_path => "#{sincedb_path}"
+              delimiter => "#{TEST_FILE_DELIMITER}"
+              codec => "json"
+            }
           }
-        }
-      CONFIG
+        CONFIG
 
-      File.open(tmpfile_path, "w") do |fd|
-        fd.puts('{"path": "my_path", "host": "my_host"}')
-        fd.puts('{"my_field": "my_val"}')
-        fd.fsync
-      end
+        File.open(tmpfile_path, "w") do |fd|
+          fd.puts('{"path": "my_path", "host": "my_host"}')
+          fd.puts('{"my_field": "my_val"}')
+          fd.fsync
+        end
 
-      events = input(conf) do |pipeline, queue|
-        2.times.collect { queue.pop }
-      end
+        events = input(conf) do |pipeline, queue|
+          2.times.collect { queue.pop }
+        end
 
-      existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
+        existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
 
-      expect(events[existing_path_index].get("path")).to eq "my_path"
-      expect(events[existing_path_index].get("host")).to eq "my_host"
-      expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+        expect(events[existing_path_index].get("path")).to eq "my_path"
+        expect(events[existing_path_index].get("host")).to eq "my_host"
+        expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
 
-      expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}"
-      expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-      expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+        expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}"
+        expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+        expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+      end
     end
 
-    it "should read old files" do
-      tmpfile_path = Stud::Temporary.pathname
-
-      conf = <<-CONFIG
-        input {
-          file {
-            type => "blah"
-            path => "#{tmpfile_path}"
-            start_position => "beginning"
-            codec => "json"
+    context "running the input twice" do
+      let(:name) { "D" }
+      it "should read old files" do
+        conf = <<-CONFIG
+          input {
+            file {
+              type => "blah"
+              path => "#{path_path}"
+              start_position => "beginning"
+              codec => "json"
+            }
           }
-        }
-      CONFIG
+        CONFIG
 
-      File.open(tmpfile_path, "w") do |fd|
-        fd.puts('{"path": "my_path", "host": "my_host"}')
-        fd.puts('{"my_field": "my_val"}')
-        fd.fsync
-      end
-      # arbitrary old file (2 days)
-      FileInput.make_file_older(tmpfile_path, 48 * 60 * 60)
+        File.open(tmpfile_path, "w") do |fd|
+          fd.puts('{"path": "my_path", "host": "my_host"}')
+          fd.puts('{"my_field": "my_val"}')
+          fd.fsync
+        end
+        # arbitrary old file (2 days)
+        FileInput.make_file_older(tmpfile_path, 48 * 60 * 60)
 
-      events = input(conf) do |pipeline, queue|
-        2.times.collect { queue.pop }
+        events = input(conf) do |pipeline, queue|
+          2.times.collect { queue.pop }
+        end
+        existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
+        expect(events[existing_path_index].get("path")).to eq "my_path"
+        expect(events[existing_path_index].get("host")).to eq "my_host"
+        expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+
+        expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}"
+        expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
+        expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
       end
-      existing_path_index, added_path_index  = "my_val" == events[0].get("my_field") ? [1,0] : [0,1]
-      expect(events[existing_path_index].get("path")).to eq "my_path"
-      expect(events[existing_path_index].get("host")).to eq "my_host"
-      expect(events[existing_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-
-      expect(events[added_path_index].get("path")).to eq "#{tmpfile_path}"
-      expect(events[added_path_index].get("host")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-      expect(events[added_path_index].get("[@metadata][host]")).to eq "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
     end
 
-    context "when sincedb_path is an existing directory" do
-      let(:tmpfile_path) { Stud::Temporary.pathname }
-      let(:sincedb_path) { Stud::Temporary.directory }
-      subject { LogStash::Inputs::File.new("path" => tmpfile_path, "sincedb_path" => sincedb_path) }
+    context "when sincedb_path is a directory" do
+      let(:name) { "E" }
+      subject { LogStash::Inputs::File.new("path" => path_path, "sincedb_path" => directory) }
 
       after :each do
         FileUtils.rm_rf(sincedb_path)
@@ -180,16 +187,19 @@ describe LogStash::Inputs::File do
   end
 
   describe "testing with new, register, run and stop" do
+    let(:suffix)       { "A" }
     let(:conf)         { Hash.new }
     let(:mlconf)       { Hash.new }
     let(:events)       { Array.new }
     let(:mlcodec)      { LogStash::Codecs::Multiline.new(mlconf) }
-    let(:codec)        { FileInput::CodecTracer.new }
-    let(:tmpfile_path) { Stud::Temporary.pathname }
-    let(:sincedb_path) { Stud::Temporary.pathname }
+    let(:tracer_codec) { FileInput::CodecTracer.new }
     let(:tmpdir_path)  { Stud::Temporary.directory }
+    let(:tmpfile_path) { ::File.join(tmpdir_path, "#{suffix}.txt") }
+    let(:path_path)    { ::File.join(tmpdir_path, "*.txt") }
+    let(:sincedb_path) { ::File.join(tmpdir_path, "sincedb-#{suffix}") }
 
     after :each do
+      sleep(0.1) until subject.completely_stopped?
       FileUtils.rm_rf(sincedb_path)
     end
 
@@ -204,7 +214,7 @@ describe LogStash::Inputs::File do
         end
         mlconf.update("pattern" => "^\s", "what" => "previous")
         conf.update("type" => "blah",
-              "path" => tmpfile_path,
+              "path" => path_path,
               "sincedb_path" => sincedb_path,
               "stat_interval" => 0.1,
               "codec" => mlcodec,
@@ -213,16 +223,22 @@ describe LogStash::Inputs::File do
 
       it "reads the appended data only" do
         subject.register
-        RSpec::Sequencing
-          .run_after(0.2, "assert zero events then append two lines") do
-            expect(events.size).to eq(0)
+        actions = RSpec::Sequencing
+          .run_after(1, "append two lines after delay") do
             File.open(tmpfile_path, "a") { |fd| fd.puts("hello"); fd.puts("world") }
           end
-          .then_after(0.4, "quit") do
+          .then("wait for one event") do
+            wait(0.75).for{events.size}.to eq(1)
+          end
+          .then("quit") do
             subject.stop
           end
+          .then("wait for flushed event") do
+            wait(0.75).for{events.size}.to eq(2)
+          end
 
         subject.run(events)
+        actions.assert_no_errors
 
         event1 = events[0]
         expect(event1).not_to be_nil
@@ -240,218 +256,172 @@ describe LogStash::Inputs::File do
 
     context "when close_older config is specified" do
       let(:line)         { "line1.1-of-a" }
-
+      let(:suffix)       { "X" }
       subject { described_class.new(conf) }
 
       before do
         conf.update(
               "type" => "blah",
-              "path" => "#{tmpdir_path}/*.log",
+              "path" => path_path,
               "sincedb_path" => sincedb_path,
               "stat_interval" => 0.02,
-              "codec" => codec,
-              "close_older" => 0.5,
+              "codec" => tracer_codec,
+              "close_older" => "100 ms",
+              "start_position" => "beginning",
               "delimiter" => TEST_FILE_DELIMITER)
 
         subject.register
       end
 
-      it "having timed_out, the identity is evicted" do
-        RSpec::Sequencing
+      it "having timed_out, the codec is auto flushed" do
+        actions = RSpec::Sequencing
           .run("create file") do
-            File.open("#{tmpdir_path}/a.log", "wb") { |file|  file.puts(line) }
+            File.open(tmpfile_path, "wb") { |file|  file.puts(line) }
           end
-          .then_after(0.3, "identity is mapped") do
-            expect(codec.trace_for(:accept)).to eq([true])
-            expect(subject.codec.identity_count).to eq(1)
+          .then_after(0.1, "identity is mapped") do
+            wait(0.75).for{subject.codec.identity_map[tmpfile_path]}.not_to be_nil, "identity is not mapped"
           end
-          .then_after(0.3, "test for auto_flush") do
-            expect(codec.trace_for(:auto_flush)).to eq([true])
-            expect(subject.codec.identity_count).to eq(0)
+          .then("wait for auto_flush") do
+            wait(0.75).for{subject.codec.identity_map[tmpfile_path].codec.trace_for(:auto_flush)}.to eq([true]), "autoflush didn't"
           end
-          .then_after(0.1, "quit") do
+          .then("quit") do
             subject.stop
           end
         subject.run(events)
+        actions.assert_no_errors
+        expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to eq([true])
       end
     end
 
     context "when ignore_older config is specified" do
-      let(:line) { "line1.1-of-a" }
-      let(:tmp_dir_file) { "#{tmpdir_path}/a.log" }
-
-      subject { described_class.new(conf) }
-
+      let(:suffix) { "Y" }
       before do
-        File.open(tmp_dir_file, "a") do |fd|
-          fd.puts(line)
-          fd.fsync
-        end
-        FileInput.make_file_older(tmp_dir_file, 2)
         conf.update(
               "type" => "blah",
-              "path" => "#{tmpdir_path}/*.log",
+              "path" => path_path,
               "sincedb_path" => sincedb_path,
               "stat_interval" => 0.02,
-              "codec" => codec,
-              "ignore_older" => 1,
+              "codec" => tracer_codec,
+              "ignore_older" => "500 ms",
               "delimiter" => TEST_FILE_DELIMITER)
-
-        subject.register
-        Thread.new { subject.run(events) }
       end
+      subject { described_class.new(conf) }
+      let(:line) { "line1.1-of-a" }
 
       it "the file is not read" do
-        sleep 0.1
-        subject.stop
-        expect(codec).to receive_call_and_args(:accept, false)
-        expect(codec).to receive_call_and_args(:auto_flush, false)
-        expect(subject.codec.identity_count).to eq(0)
+        subject.register
+        RSpec::Sequencing
+          .run("create file") do
+            File.open(tmp_dir_file, "a") do |fd|
+              fd.puts(line)
+              fd.fsync
+            end
+            FileInput.make_file_older(tmp_dir_file, 2)
+          end
+          .then_after(0.5, "stop") do
+            subject.stop
+          end
+        subject.run(events)
+        expect(subject.codec.identity_map[tmpfile_path].codec.trace_for(:accept)).to be_falsey
       end
     end
 
     context "when wildcard path and a multiline codec is specified" do
       subject { described_class.new(conf) }
-
+      let(:suffix)       { "J" }
+      let(:tmpfile_path2) { ::File.join(tmpdir_path, "K.txt") }
       before do
         mlconf.update("pattern" => "^\s", "what" => "previous")
         conf.update(
               "type" => "blah",
-              "path" => "#{tmpdir_path}/*.log",
+              "path" => path_path,
+              "start_position" => "beginning",
               "sincedb_path" => sincedb_path,
               "stat_interval" => 0.05,
               "codec" => mlcodec,
+              "file_sort_by" => "path",
               "delimiter" => TEST_FILE_DELIMITER)
 
         subject.register
       end
 
       it "collects separate multiple line events from each file" do
+        subject
         actions = RSpec::Sequencing
           .run_after(0.1, "create files") do
-            File.open("#{tmpdir_path}/A.log", "wb") do |fd|
-              fd.puts("line1.1-of-a")
-              fd.puts("  line1.2-of-a")
-              fd.puts("  line1.3-of-a")
+            File.open(tmpfile_path, "wb") do |fd|
+              fd.puts("line1.1-of-J")
+              fd.puts("  line1.2-of-J")
+              fd.puts("  line1.3-of-J")
             end
-            File.open("#{tmpdir_path}/z.log", "wb") do |fd|
-              fd.puts("line1.1-of-z")
-              fd.puts("  line1.2-of-z")
-              fd.puts("  line1.3-of-z")
+            File.open(tmpfile_path2, "wb") do |fd|
+              fd.puts("line1.1-of-K")
+              fd.puts("  line1.2-of-K")
+              fd.puts("  line1.3-of-K")
             end
           end
-          .then_after(0.2, "assert both files are mapped as identities and stop") do
-            expect(subject.codec.identity_count).to eq(2)
+          .then("assert both files are mapped as identities and stop") do
+            wait(2).for {subject.codec.identity_count}.to eq(2), "both files are not mapped as identities"
           end
-          .then_after(0.1, "stop") do
+          .then("stop") do
             subject.stop
           end
-          .then_after(0.2 , "stop flushes both events") do
-            expect(events.size).to eq(2)
-            e1, e2 = events
-            e1_message = e1.get("message")
-            e2_message = e2.get("message")
-
-            # can't assume File A will be read first
-            if e1_message.start_with?('line1.1-of-z')
-              expect(e1.get("path")).to match(/z.log/)
-              expect(e2.get("path")).to match(/A.log/)
-              expect(e1_message).to eq("line1.1-of-z#{TEST_FILE_DELIMITER}  line1.2-of-z#{TEST_FILE_DELIMITER}  line1.3-of-z")
-              expect(e2_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER}  line1.2-of-a#{TEST_FILE_DELIMITER}  line1.3-of-a")
-            else
-              expect(e1.get("path")).to match(/A.log/)
-              expect(e2.get("path")).to match(/z.log/)
-              expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER}  line1.2-of-a#{TEST_FILE_DELIMITER}  line1.3-of-a")
-              expect(e2_message).to eq("line1.1-of-z#{TEST_FILE_DELIMITER}  line1.2-of-z#{TEST_FILE_DELIMITER}  line1.3-of-z")
-            end
-          end
         subject.run(events)
         # wait for actions to complete
-        actions.value
+        actions.assert_no_errors
+        expect(events.size).to eq(2)
+        e1, e2 = events
+        e1_message = e1.get("message")
+        e2_message = e2.get("message")
+
+        expect(e1.get("path")).to match(/J.txt/)
+        expect(e2.get("path")).to match(/K.txt/)
+        expect(e1_message).to eq("line1.1-of-J#{TEST_FILE_DELIMITER}  line1.2-of-J#{TEST_FILE_DELIMITER}  line1.3-of-J")
+        expect(e2_message).to eq("line1.1-of-K#{TEST_FILE_DELIMITER}  line1.2-of-K#{TEST_FILE_DELIMITER}  line1.3-of-K")
       end
 
       context "if auto_flush is enabled on the multiline codec" do
         let(:mlconf) { { "auto_flush_interval" => 0.5 } }
-
+        let(:suffix)       { "M" }
         it "an event is generated via auto_flush" do
           actions = RSpec::Sequencing
             .run_after(0.1, "create files") do
-              File.open("#{tmpdir_path}/A.log", "wb") do |fd|
+              File.open(tmpfile_path, "wb") do |fd|
                 fd.puts("line1.1-of-a")
                 fd.puts("  line1.2-of-a")
                 fd.puts("  line1.3-of-a")
               end
             end
-            .then_after(0.75, "wait for auto_flush") do
-              e1 = events.first
-              e1_message = e1.get("message")
-              expect(e1["path"]).to match(/a.log/)
-              expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER}  line1.2-of-a#{TEST_FILE_DELIMITER}  line1.3-of-a")
+            .then("wait for auto_flush") do
+              wait(2).for{events.size}.to eq(1), "events size is not 1"
             end
             .then("stop") do
               subject.stop
             end
           subject.run(events)
           # wait for actions to complete
-          actions.value
+          actions.assert_no_errors
+          e1 = events.first
+          e1_message = e1.get("message")
+          expect(e1_message).to eq("line1.1-of-a#{TEST_FILE_DELIMITER}  line1.2-of-a#{TEST_FILE_DELIMITER}  line1.3-of-a")
+          expect(e1.get("path")).to match(/M.txt$/)
         end
       end
     end
 
-    context "when #run is called multiple times", :unix => true do
-      let(:file_path)    { "#{tmpdir_path}/a.log" }
-      let(:buffer)       { [] }
-      let(:run_thread_proc) do
-        lambda { Thread.new { subject.run(buffer) } }
-      end
-      let(:lsof_proc) do
-        lambda { `lsof -p #{Process.pid} | grep #{file_path}` }
-      end
-
-      subject { described_class.new(conf) }
-
-      before do
-        conf.update(
-          "path" => tmpdir_path + "/*.log",
-          "start_position" => "beginning",
-          "stat_interval" => 0.1,
-          "sincedb_path" => sincedb_path)
-
-        File.open(file_path, "w") do |fd|
-          fd.puts('foo')
-          fd.puts('bar')
-          fd.fsync
-        end
-      end
-
-      it "should only actually open files when content changes are detected" do
-        subject.register
-        expect(lsof_proc.call).to eq("")
-        # first run processes the file and records sincedb progress
-        run_thread_proc.call
-        wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(1)
-        # second run quits the first run
-        # sees the file has not changed size and does not open it
-        run_thread_proc.call
-        wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(0)
-        # truncate and write less than before
-        File.open(file_path, "w"){ |fd| fd.puts('baz'); fd.fsync }
-        # sees the file has changed size and does open it
-        wait(1).for{lsof_proc.call.scan(file_path).size}.to eq(1)
-      end
-    end
-
     describe "specifying max_open_files" do
+      let(:suffix)       { "P" }
+      let(:tmpfile_path2) { ::File.join(tmpdir_path, "Q.txt") }
       subject { described_class.new(conf) }
       before do
-        File.open("#{tmpdir_path}/a.log", "w") do |fd|
-          fd.puts("line1-of-a")
-          fd.puts("line2-of-a")
+        File.open(tmpfile_path, "w") do |fd|
+          fd.puts("line1-of-P")
+          fd.puts("line2-of-P")
           fd.fsync
         end
-        File.open("#{tmpdir_path}/z.log", "w") do |fd|
-          fd.puts("line1-of-z")
-          fd.puts("line2-of-z")
+        File.open(tmpfile_path2, "w") do |fd|
+          fd.puts("line1-of-Q")
+          fd.puts("line2-of-Q")
           fd.fsync
         end
       end
@@ -461,37 +431,32 @@ describe LogStash::Inputs::File do
           conf.clear
           conf.update(
                 "type" => "blah",
-                "path" => "#{tmpdir_path}/*.log",
+                "path" => path_path,
                 "sincedb_path" => sincedb_path,
                 "stat_interval" => 0.1,
                 "max_open_files" => 1,
                 "start_position" => "beginning",
+                "file_sort_by" => "path",
                 "delimiter" => TEST_FILE_DELIMITER)
           subject.register
         end
         it "collects line events from only one file" do
           actions = RSpec::Sequencing
-            .run_after(0.2, "assert one identity is mapped") do
-              expect(subject.codec.identity_count).to eq(1)
+            .run("assert one identity is mapped") do
+              wait(0.4).for{subject.codec.identity_count}.to be > 0, "no identity is mapped"
             end
-            .then_after(0.1, "stop") do
+            .then("stop") do
               subject.stop
             end
-            .then_after(0.1, "stop flushes last event") do
-              expect(events.size).to eq(2)
-              e1, e2 = events
-              if Dir.glob("#{tmpdir_path}/*.log").first =~ %r{a\.log}
-                #linux and OSX have different retrieval order
-                expect(e1.get("message")).to eq("line1-of-a")
-                expect(e2.get("message")).to eq("line2-of-a")
-              else
-                expect(e1.get("message")).to eq("line1-of-z")
-                expect(e2.get("message")).to eq("line2-of-z")
-              end
+            .then("stop flushes last event") do
+              wait(0.4).for{events.size}.to eq(2), "events size does not equal 2"
             end
           subject.run(events)
           # wait for actions future value
-          actions.value
+          actions.assert_no_errors
+          e1, e2 = events
+          expect(e1.get("message")).to eq("line1-of-P")
+          expect(e2.get("message")).to eq("line2-of-P")
         end
       end
 
@@ -499,41 +464,36 @@ describe LogStash::Inputs::File do
         before do
           conf.update(
                 "type" => "blah",
-                "path" => "#{tmpdir_path}/*.log",
+                "path" => path_path,
                 "sincedb_path" => sincedb_path,
                 "stat_interval" => 0.1,
                 "max_open_files" => 1,
                 "close_older" => 0.5,
                 "start_position" => "beginning",
+                "file_sort_by" => "path",
                 "delimiter" => TEST_FILE_DELIMITER)
           subject.register
         end
 
         it "collects line events from both files" do
           actions = RSpec::Sequencing
-            .run_after(0.2, "assert both identities are mapped and the first two events are built") do
-              expect(subject.codec.identity_count).to eq(2)
-              expect(events.size).to eq(2)
+            .run("assert both identities are mapped and the first two events are built") do
+              wait(0.4).for{subject.codec.identity_count == 1 && events.size == 2}.to eq(true), "both identities are not mapped and the first two events are not built"
             end
-            .then_after(0.8, "wait for close to flush last event of each identity") do
-              expect(events.size).to eq(4)
-              if Dir.glob("#{tmpdir_path}/*.log").first =~ %r{a\.log}
-                #linux and OSX have different retrieval order
-                e1, e2, e3, e4 = events
-              else
-                e3, e4, e1, e2 = events
-              end
-              expect(e1.get("message")).to eq("line1-of-a")
-              expect(e2.get("message")).to eq("line2-of-a")
-              expect(e3.get("message")).to eq("line1-of-z")
-              expect(e4.get("message")).to eq("line2-of-z")
+            .then("wait for close to flush last event of each identity") do
+              wait(0.8).for{events.size}.to eq(4), "close does not flush last event of each identity"
             end
             .then_after(0.1, "stop") do
               subject.stop
             end
           subject.run(events)
           # wait for actions future value
-          actions.value
+          actions.assert_no_errors
+          e1, e2, e3, e4 = events
+          expect(e1.get("message")).to eq("line1-of-P")
+          expect(e2.get("message")).to eq("line2-of-P")
+          expect(e3.get("message")).to eq("line1-of-Q")
+          expect(e4.get("message")).to eq("line2-of-Q")
         end
       end
     end