logstash-input-file 2.0.3 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5377f5857709a60359d657a8007d6ab0b3f87f4d
-  data.tar.gz: 1af3cfea2b4135413c3cb347e61b7c0ff89985cc
+  metadata.gz: 5447c5eea15c163ea16990bd7362f52081d665bb
+  data.tar.gz: 0768e19af5a4694fbe8cad1d143d03db42321099
 SHA512:
-  metadata.gz: 2bd62efe503f31050170ebc8f17455f8c108bb0e0430a7ee1718250831292a90bf218bec728a418aa0cb3644cd587f50dadb35ce51ed6b5256e353bfe90d143b
-  data.tar.gz: 69ecdc00558c2b9da3a1d7e989e1a4936e0c24ab4a7f1c420bc64ba2dbd460f6751014821f71e25263b57dbd8e4598e637688233eee9233587ac73ceb58d776c
+  metadata.gz: 105bfd207188e1625d888fee6515ad25f1d8c9abdc3309f5f70ddfb431c989611d3ca60e18d9b39feff39578312cba74e0ab69ba40e26c4e96a9d657713ea448
+  data.tar.gz: 080fd1a1aca5a2c199d0d3c0a86096cbce28dc60e7be7be83bb2612838d0d0c6c3137bc22c7f8397a9749638e368ef2dc82f56a2c7672bfeddc576e0bf81bdef

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.1.0
+ - Implement new config options: ignore_older and close_older.  When close_older is set, any buffered data will be flushed.
+ - Fixes [#81](https://github.com/logstash-plugins/logstash-input-file/issues/81)
+ - Fixes [#81](https://github.com/logstash-plugins/logstash-input-file/issues/89)
+ - Fixes [#81](https://github.com/logstash-plugins/logstash-input-file/issues/90)
+
 ## 2.0.3
  - Implement Stream Identity mapping of codecs: distinct codecs will collect input per stream identity (filename)
 

data/Gemfile

@@ -1,2 +1,2 @@
 source 'https://rubygems.org'
-gemspec
+gemspec

data/lib/logstash/inputs/file.rb

@@ -62,15 +62,29 @@ require "socket" # for Socket.gethostname
 # to the rotation and its reopening under the new name (an interval
 # determined by the `stat_interval` and `discover_interval` options)
 # will not get picked up.
+
+class LogStash::Codecs::Base
+  # TODO - move this to core
+  if !method_defined?(:accept)
+    def accept(listener)
+      decode(listener.data) do |event|
+        listener.process_event(event)
+      end
+    end
+  end
+  if !method_defined?(:auto_flush)
+    def auto_flush
+    end
+  end
+end
+
 class LogStash::Inputs::File < LogStash::Inputs::Base
   config_name "file"
 
-  # TODO(sissel): This should switch to use the `line` codec by default
-  # once file following
-  default :codec, "plain"
-
   # The path(s) to the file(s) to use as an input.
   # You can use filename patterns here, such as `/var/log/*.log`.
+  # If you use a pattern like `/var/log/**/*.log`, a recursive search
+  # of `/var/log` will be done for all `*.log` files.
   # Paths must be absolute and cannot be relative.
   #
   # You may also configure multiple paths. See an example
@@ -121,6 +135,17 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   # set the new line delimiter, defaults to "\n"
   config :delimiter, :validate => :string, :default => "\n"
 
+  # If this option is specified, when the file input discovers a file that
+  # was last modified before the specified timespan in seconds, the file is
+  # ignored. After it's discovery, if an ignored file is modified it is no
+  # longer ignored and any new data is read. The default is 24 hours.
+  config :ignore_older, :validate => :number, :default => 24 * 60 * 60
+
+  # If this option is specified, the file input closes any files that remain
+  # unmodified for longer than the specified timespan in seconds.
+  # The default is 1 hour
+  config :close_older, :validate => :number, :default => 1 * 60 * 60
+
   public
   def register
     require "addressable/uri"
@@ -135,7 +160,8 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
       :discover_interval => @discover_interval,
       :sincedb_write_interval => @sincedb_write_interval,
       :delimiter => @delimiter,
-      :logger => @logger,
+      :ignore_older => @ignore_older,
+      :close_older => @close_older
     }
 
     @path.each do |path|
@@ -186,36 +212,84 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
     @codec = LogStash::Codecs::IdentityMapCodec.new(@codec)
   end # def register
 
-  def run(queue)
-    @tail = FileWatch::Tail.new(@tail_config)
+  class ListenerTail
+    # use attr_reader to define noop methods
+    attr_reader :input, :path, :data
+    attr_reader :deleted, :created, :error, :eof
+
+    # construct with upstream state
+    def initialize(path, input)
+      @path, @input = path, input
+    end
+
+    def timed_out
+      input.codec.evict(path)
+    end
+
+    def accept(data)
+      # and push transient data filled dup listener downstream
+      input.log_line_received(path, data)
+      input.codec.accept(dup_adding_state(data))
+    end
+
+    def process_event(event)
+      event["[@metadata][path]"] = path
+      event["path"] = path if !event.include?("path")
+      input.post_process_this(event)
+    end
+
+    def add_state(data)
+      @data = data
+      self
+    end
+
+    private
+
+    # duplicate and add state for downstream
+    def dup_adding_state(line)
+      self.class.new(path, input).add_state(line)
+    end
+  end
+
+  def listener_for(path)
+    # path is the identity
+    ListenerTail.new(path, self)
+  end
+
+  def begin_tailing
+    # if the pipeline restarts this input,
+    # make sure previous files are closed
+    stop
+    # use observer listener api
+    @tail = FileWatch::Tail.new_observing(@tail_config)
     @tail.logger = @logger
     @path.each { |path| @tail.tail(path) }
-    @tail.subscribe do |path, line|
-      log_line_received(path, line)
-      @codec.decode(line, path) do |event|
-        # path is the identity
-        # Note: this block is cached in the
-        # identity_map_codec for use when
-        # buffered lines are flushed.
-        queue << add_path_meta(event, path)
-      end
-    end
+  end
+
+  def run(queue)
+    begin_tailing
+    @queue = queue
+    @tail.subscribe(self)
   end # def run
 
+  def post_process_this(event)
+    event["host"] = @host if !event.include?("host")
+    decorate(event)
+    @queue << event
+  end
+
   def log_line_received(path, line)
     return if !@logger.debug?
     @logger.debug("Received line", :path => path, :text => line)
   end
 
-  def add_path_meta(event, path)
-    event["[@metadata][path]"] = path
-    event["host"] = @host if !event.include?("host")
-    event["path"] = path if !event.include?("path")
-    decorate(event)
-    event
-  end
-
   def stop
-    @tail.quit if @tail # _sincedb_write is called implicitly
+    # in filewatch >= 0.6.7, quit will closes and forget all files
+    # but it will write their last read positions to since_db
+    # beforehand
+    if @tail
+      @codec.close
+      @tail.quit
+    end
   end
 end # class LogStash::Inputs::File

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '2.0.3'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,12 +20,12 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0.alpha0"
 
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'addressable'
-  s.add_runtime_dependency 'filewatch', ['>= 0.6.5', '~> 0.6']
-  s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.3']
+  s.add_runtime_dependency 'filewatch', ['>= 0.7.0', '~> 0.7']
+  s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.5']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
   s.add_development_dependency 'logstash-devutils'

data/spec/inputs/file_spec.rb

@@ -1,256 +1,394 @@
 # encoding: utf-8
 
-require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/file"
+require_relative "../spec_helper"
 require "tempfile"
 require "stud/temporary"
-require "logstash/inputs/file"
+require "logstash/codecs/multiline"
 
 FILE_DELIMITER = LogStash::Environment.windows? ? "\r\n" : "\n"
 
 describe LogStash::Inputs::File do
-
-  it_behaves_like "an interruptible input plugin" do
-    let(:config) do
-      {
-        "path" => Stud::Temporary.pathname,
-        "sincedb_path" => Stud::Temporary.pathname
-      }
+  describe "testing with input(conf) do |pipeline, queue|" do
+    it_behaves_like "an interruptible input plugin" do
+      let(:config) do
+        {
+          "path" => Stud::Temporary.pathname,
+          "sincedb_path" => Stud::Temporary.pathname
+        }
+      end
     end
-  end
 
-  it "should starts at the end of an existing file" do
-    tmpfile_path = Stud::Temporary.pathname
-    sincedb_path = Stud::Temporary.pathname
-
-    conf = <<-CONFIG
-      input {
-        file {
-          type => "blah"
-          path => "#{tmpfile_path}"
-          sincedb_path => "#{sincedb_path}"
-          delimiter => "#{FILE_DELIMITER}"
+    it "should start at the beginning of an existing file" do
+      tmpfile_path = Stud::Temporary.pathname
+      sincedb_path = Stud::Temporary.pathname
+
+      conf = <<-CONFIG
+        input {
+          file {
+            type => "blah"
+            path => "#{tmpfile_path}"
+            start_position => "beginning"
+            sincedb_path => "#{sincedb_path}"
+            delimiter => "#{FILE_DELIMITER}"
+          }
         }
-      }
-    CONFIG
+      CONFIG
 
-    File.open(tmpfile_path, "w") do |fd|
-      fd.puts("ignore me 1")
-      fd.puts("ignore me 2")
+      File.open(tmpfile_path, "a") do |fd|
+        fd.puts("hello")
+        fd.puts("world")
+        fd.fsync
+      end
+
+      events = input(conf) do |pipeline, queue|
+        2.times.collect { queue.pop }
+      end
+
+      insist { events[0]["message"] } == "hello"
+      insist { events[1]["message"] } == "world"
     end
 
-    events = input(conf) do |pipeline, queue|
+    it "should restarts at the sincedb value" do
+      tmpfile_path = Stud::Temporary.pathname
+      sincedb_path = Stud::Temporary.pathname
 
-      # at this point the plugins
-      # threads might still be initializing so we cannot know when the
-      # file plugin will have seen the original file, it could see it
-      # after the first(s) hello world appends below, hence the
-      # retry logic.
+      conf = <<-CONFIG
+        input {
+          file {
+            type => "blah"
+            path => "#{tmpfile_path}"
+            start_position => "beginning"
+            sincedb_path => "#{sincedb_path}"
+            delimiter => "#{FILE_DELIMITER}"
+          }
+        }
+      CONFIG
 
-      events = []
+      File.open(tmpfile_path, "w") do |fd|
+        fd.puts("hello3")
+        fd.puts("world3")
+      end
 
-      retries = 0
-      while retries < 20
-        File.open(tmpfile_path, "a") do |fd|
-          fd.puts("hello")
-          fd.puts("world")
-        end
+      events = input(conf) do |pipeline, queue|
+        2.times.collect { queue.pop }
+      end
 
-        if queue.size >= 2
-          events = 2.times.collect { queue.pop }
-          break
-        end
+      insist { events[0]["message"] } == "hello3"
+      insist { events[1]["message"] } == "world3"
+
+      File.open(tmpfile_path, "a") do |fd|
+        fd.puts("foo")
+        fd.puts("bar")
+        fd.puts("baz")
+        fd.fsync
+      end
 
-        sleep(0.1)
-        retries += 1
+      events = input(conf) do |pipeline, queue|
+        3.times.collect { queue.pop }
       end
 
-      events
+      insist { events[0]["message"] } == "foo"
+      insist { events[1]["message"] } == "bar"
+      insist { events[2]["message"] } == "baz"
     end
 
-    insist { events[0]["message"] } == "hello"
-    insist { events[1]["message"] } == "world"
-  end
+    it "should not overwrite existing path and host fields" do
+      tmpfile_path = Stud::Temporary.pathname
+      sincedb_path = Stud::Temporary.pathname
 
-  it "should start at the beginning of an existing file" do
-    tmpfile_path = Stud::Temporary.pathname
-    sincedb_path = Stud::Temporary.pathname
-
-    conf = <<-CONFIG
-      input {
-        file {
-          type => "blah"
-          path => "#{tmpfile_path}"
-          start_position => "beginning"
-          sincedb_path => "#{sincedb_path}"
-          delimiter => "#{FILE_DELIMITER}"
+      conf = <<-CONFIG
+        input {
+          file {
+            type => "blah"
+            path => "#{tmpfile_path}"
+            start_position => "beginning"
+            sincedb_path => "#{sincedb_path}"
+            delimiter => "#{FILE_DELIMITER}"
+            codec => "json"
+          }
         }
-      }
-    CONFIG
-
-    File.open(tmpfile_path, "a") do |fd|
-      fd.puts("hello")
-      fd.puts("world")
-    end
+      CONFIG
 
-    events = input(conf) do |pipeline, queue|
-      2.times.collect { queue.pop }
-    end
+      File.open(tmpfile_path, "w") do |fd|
+        fd.puts('{"path": "my_path", "host": "my_host"}')
+        fd.puts('{"my_field": "my_val"}')
+        fd.fsync
+      end
 
-    insist { events[0]["message"] } == "hello"
-    insist { events[1]["message"] } == "world"
-  end
+      events = input(conf) do |pipeline, queue|
+        2.times.collect { queue.pop }
+      end
 
-  it "should restarts at the sincedb value" do
-    tmpfile_path = Stud::Temporary.pathname
-    sincedb_path = Stud::Temporary.pathname
-
-    conf = <<-CONFIG
-      input {
-        file {
-          type => "blah"
-          path => "#{tmpfile_path}"
-          start_position => "beginning"
-          sincedb_path => "#{sincedb_path}"
-          delimiter => "#{FILE_DELIMITER}"
-        }
-      }
-    CONFIG
+      insist { events[0]["path"] } == "my_path"
+      insist { events[0]["host"] } == "my_host"
 
-    File.open(tmpfile_path, "w") do |fd|
-      fd.puts("hello3")
-      fd.puts("world3")
+      insist { events[1]["path"] } == "#{tmpfile_path}"
+      insist { events[1]["host"] } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
     end
 
-    events = input(conf) do |pipeline, queue|
-      2.times.collect { queue.pop }
-    end
+    context "when sincedb_path is an existing directory" do
+      let(:tmpfile_path) { Stud::Temporary.pathname }
+      let(:sincedb_path) { Stud::Temporary.directory }
+      subject { LogStash::Inputs::File.new("path" => tmpfile_path, "sincedb_path" => sincedb_path) }
 
-    insist { events[0]["message"] } == "hello3"
-    insist { events[1]["message"] } == "world3"
+      after :each do
+        FileUtils.rm_rf(sincedb_path)
+      end
 
-    File.open(tmpfile_path, "a") do |fd|
-      fd.puts("foo")
-      fd.puts("bar")
-      fd.puts("baz")
+      it "should raise exception" do
+        expect { subject.register }.to raise_error(ArgumentError)
+      end
     end
+  end
 
-    events = input(conf) do |pipeline, queue|
-      3.times.collect { queue.pop }
-    end
+  describe "testing with new, register, run and stop" do
+    let(:conf)         { Hash.new }
+    let(:mlconf)       { Hash.new }
+    let(:events)       { Array.new }
+    let(:mlcodec)      { LogStash::Codecs::Multiline.new(mlconf) }
+    let(:codec)        { CodecTracer.new }
+    let(:tmpfile_path) { Stud::Temporary.pathname }
+    let(:sincedb_path) { Stud::Temporary.pathname }
+    let(:tmpdir_path)  { Stud::Temporary.directory }
 
-    insist { events[0]["message"] } == "foo"
-    insist { events[1]["message"] } == "bar"
-    insist { events[2]["message"] } == "baz"
-  end
+    context "when data exists and then more data is appended" do
+      subject { described_class.new(conf) }
 
-  it "should not overwrite existing path and host fields" do
-    tmpfile_path = Stud::Temporary.pathname
-    sincedb_path = Stud::Temporary.pathname
-
-    conf = <<-CONFIG
-      input {
-        file {
-          type => "blah"
-          path => "#{tmpfile_path}"
-          start_position => "beginning"
-          sincedb_path => "#{sincedb_path}"
-          delimiter => "#{FILE_DELIMITER}"
-          codec => "json"
-        }
-      }
-    CONFIG
+      before do
+        File.open(tmpfile_path, "w") do |fd|
+          fd.puts("ignore me 1")
+          fd.puts("ignore me 2")
+          fd.fsync
+        end
+        mlconf.update("pattern" => "^\s", "what" => "previous")
+        conf.update("type" => "blah",
+              "path" => tmpfile_path,
+              "sincedb_path" => sincedb_path,
+              "stat_interval" => 0.1,
+              "codec" => mlcodec,
+              "delimiter" => FILE_DELIMITER)
+        subject.register
+        Thread.new { subject.run(events) }
+      end
 
-    File.open(tmpfile_path, "w") do |fd|
-      fd.puts('{"path": "my_path", "host": "my_host"}')
-      fd.puts('{"my_field": "my_val"}')
+      it "reads the appended data only" do
+        sleep 0.1
+        File.open(tmpfile_path, "a") do |fd|
+          fd.puts("hello")
+          fd.puts("world")
+          fd.fsync
+        end
+        # wait for one event, the last line is buffered
+        expect(pause_until{ events.size == 1 }).to be_truthy
+        subject.stop
+        # stop flushes the second event
+        expect(pause_until{ events.size == 2 }).to be_truthy
+
+        event1 = events[0]
+        expect(event1).not_to be_nil
+        expect(event1["path"]).to eq tmpfile_path
+        expect(event1["@metadata"]["path"]).to eq tmpfile_path
+        expect(event1["message"]).to eq "hello"
+
+        event2 = events[1]
+        expect(event2).not_to be_nil
+        expect(event2["path"]).to eq tmpfile_path
+        expect(event2["@metadata"]["path"]).to eq tmpfile_path
+        expect(event2["message"]).to eq "world"
+      end
     end
 
-    events = input(conf) do |pipeline, queue|
-      2.times.collect { queue.pop }
-    end
+    context "when close_older config is specified" do
+      let(:line)         { "line1.1-of-a" }
 
-    insist { events[0]["path"] } == "my_path"
-    insist { events[0]["host"] } == "my_host"
+      subject { described_class.new(conf) }
 
-    insist { events[1]["path"] } == "#{tmpfile_path}"
-    insist { events[1]["host"] } == "#{Socket.gethostname.force_encoding(Encoding::UTF_8)}"
-  end
+      before do
+        conf.update(
+              "type" => "blah",
+              "path" => "#{tmpdir_path}/*.log",
+              "sincedb_path" => sincedb_path,
+              "stat_interval" => 0.02,
+              "codec" => codec,
+              "close_older" => 1,
+              "delimiter" => FILE_DELIMITER)
 
-  context "when sincedb_path is an existing directory" do
-    let(:tmpfile_path) { Stud::Temporary.pathname }
-    let(:sincedb_path) { Stud::Temporary.directory }
-    subject { LogStash::Inputs::File.new("path" => tmpfile_path, "sincedb_path" => sincedb_path) }
+        subject.register
+        Thread.new { subject.run(events) }
+      end
 
-    after :each do
-      FileUtils.rm_rf(sincedb_path)
+      it "having timed_out, the identity is evicted" do
+        sleep 0.1
+        File.open("#{tmpdir_path}/a.log", "a") do |fd|
+          fd.puts(line)
+          fd.fsync
+        end
+        expect(pause_until{ subject.codec.identity_count == 1 }).to be_truthy
+        expect(codec).to receive_call_and_args(:accept, [true])
+        # wait for expiry to kick in and close files.
+        expect(pause_until{ subject.codec.identity_count.zero? }).to be_truthy
+        expect(codec).to receive_call_and_args(:auto_flush, [true])
+        subject.stop
+      end
     end
 
-    it "should raise exception" do
-      expect { subject.register }.to raise_error(ArgumentError)
-    end
-  end
+    context "when ignore_older config is specified" do
+      let(:line) { "line1.1-of-a" }
 
-  context "when wildcard path and a multiline codec is specified" do
-    let(:tmpdir_path)  { Stud::Temporary.directory }
-    let(:sincedb_path) { Stud::Temporary.pathname }
-    let(:conf) do
-      <<-CONFIG
-        input {
-          file {
-            type => "blah"
-            path => "#{tmpdir_path}/*.log"
-            start_position => "beginning"
-            sincedb_path => "#{sincedb_path}"
-            delimiter => "#{FILE_DELIMITER}"
-            codec => multiline { pattern => "^\s" what => previous }
-          }
-        }
-      CONFIG
-    end
+      subject { described_class.new(conf) }
 
-    let(:writer_proc) do
-      -> do
+      before do
         File.open("#{tmpdir_path}/a.log", "a") do |fd|
-          fd.puts("line1.1-of-a")
-          fd.puts("  line1.2-of-a")
-          fd.puts("  line1.3-of-a")
-          fd.puts("line2.1-of-a")
+          fd.puts(line)
+          fd.fsync
         end
+        sleep 1.1 # wait for file to age
+        conf.update(
+              "type" => "blah",
+              "path" => "#{tmpdir_path}/*.log",
+              "sincedb_path" => sincedb_path,
+              "stat_interval" => 0.02,
+              "codec" => codec,
+              "ignore_older" => 1,
+              "delimiter" => FILE_DELIMITER)
+
+        subject.register
+        Thread.new { subject.run(events) }
+      end
 
-        File.open("#{tmpdir_path}/z.log", "a") do |fd|
-          fd.puts("line1.1-of-z")
-          fd.puts("  line1.2-of-z")
-          fd.puts("  line1.3-of-z")
-          fd.puts("line2.1-of-z")
-        end
+      it "the file is not read" do
+        sleep 0.5
+        subject.stop
+        expect(codec).to receive_call_and_args(:accept, false)
+        expect(codec).to receive_call_and_args(:auto_flush, false)
+        expect(subject.codec.identity_count).to eq(0)
       end
     end
 
-    after do
-      FileUtils.rm_rf(tmpdir_path)
-    end
+    context "when wildcard path and a multiline codec is specified" do
+      subject { described_class.new(conf) }
+      let(:writer_proc) do
+        -> do
+          File.open("#{tmpdir_path}/a.log", "a") do |fd|
+            fd.puts("line1.1-of-a")
+            fd.puts("  line1.2-of-a")
+            fd.puts("  line1.3-of-a")
+            fd.fsync
+          end
+          File.open("#{tmpdir_path}/z.log", "a") do |fd|
+            fd.puts("line1.1-of-z")
+            fd.puts("  line1.2-of-z")
+            fd.puts("  line1.3-of-z")
+            fd.fsync
+          end
+        end
+      end
 
-    let(:event_count) { 2 }
+      before do
+        mlconf.update("pattern" => "^\s", "what" => "previous")
+        conf.update(
+              "type" => "blah",
+              "path" => "#{tmpdir_path}/*.log",
+              "sincedb_path" => sincedb_path,
+              "stat_interval" => 0.05,
+              "codec" => mlcodec,
+              "delimiter" => FILE_DELIMITER)
+
+        subject.register
+        Thread.new { subject.run(events) }
+        sleep 0.1
+        writer_proc.call
+      end
 
-    it "collects separate multiple line events from each file" do
-      writer_proc.call
+      it "collects separate multiple line events from each file" do
+        # wait for both paths to be mapped as identities
+        expect(pause_until{ subject.codec.identity_count == 2 }).to be_truthy
+        subject.stop
+        # stop flushes both events
+        expect(pause_until{ events.size == 2 }).to be_truthy
+
+        e1, e2 = events
+        e1_message = e1["message"]
+        e2_message = e2["message"]
+
+        # can't assume File A will be read first
+        if e1_message.start_with?('line1.1-of-z')
+          expect(e1["path"]).to match(/z.log/)
+          expect(e2["path"]).to match(/a.log/)
+          expect(e1_message).to eq("line1.1-of-z#{FILE_DELIMITER}  line1.2-of-z#{FILE_DELIMITER}  line1.3-of-z")
+          expect(e2_message).to eq("line1.1-of-a#{FILE_DELIMITER}  line1.2-of-a#{FILE_DELIMITER}  line1.3-of-a")
+        else
+          expect(e1["path"]).to match(/a.log/)
+          expect(e2["path"]).to match(/z.log/)
+          expect(e1_message).to eq("line1.1-of-a#{FILE_DELIMITER}  line1.2-of-a#{FILE_DELIMITER}  line1.3-of-a")
+          expect(e2_message).to eq("line1.1-of-z#{FILE_DELIMITER}  line1.2-of-z#{FILE_DELIMITER}  line1.3-of-z")
+        end
+      end
 
-      events = input(conf) do |pipeline, queue|
-        queue.size.times.collect { queue.pop }
+      context "if auto_flush is enabled on the multiline codec" do
+        let(:writer_proc) do
+          -> do
+            File.open("#{tmpdir_path}/a.log", "a") do |fd|
+              fd.puts("line1.1-of-a")
+              fd.puts("  line1.2-of-a")
+              fd.puts("  line1.3-of-a")
+            end
+          end
+        end
+        let(:mlconf) { { "auto_flush_interval" => 1 } }
+
+        it "an event is generated via auto_flush" do
+          # wait for auto_flush
+          # without it lines are buffered and pause_until would time out i.e false
+          expect(pause_until{ events.size == 1 }).to be_truthy
+          subject.stop
+
+          e1 = events.first
+          e1_message = e1["message"]
+          expect(e1["path"]).to match(/a.log/)
+          expect(e1_message).to eq("line1.1-of-a#{FILE_DELIMITER}  line1.2-of-a#{FILE_DELIMITER}  line1.3-of-a")
+        end
+      end
+    end
+
+    context "when #run is called multiple times", :unix => true do
+      let(:file_path)    { "#{tmpdir_path}/a.log" }
+      let(:buffer)       { [] }
+      let(:lsof)         { [] }
+      let(:run_thread_proc) do
+        lambda { Thread.new { subject.run(buffer) } }
+      end
+      let(:lsof_proc) do
+        lambda { `lsof -p #{Process.pid} | grep #{file_path}` }
       end
 
-      expect(events.size).to eq(event_count)
+      subject { described_class.new(conf) }
 
-      e1_message = events[0]["message"]
-      e2_message = events[1]["message"]
+      before do
+        conf.update(
+          "path" => tmpdir_path + "/*.log",
+          "start_position" => "beginning",
+          "sincedb_path" => sincedb_path)
+
+        File.open(file_path, "w") do |fd|
+          fd.puts('foo')
+          fd.puts('bar')
+          fd.fsync
+        end
+      end
 
-      # can't assume File A will be read first
-      if e1_message.start_with?('line1.1-of-z')
-        expect(e1_message).to eq("line1.1-of-z#{FILE_DELIMITER}  line1.2-of-z#{FILE_DELIMITER}  line1.3-of-z")
-        expect(e2_message).to eq("line1.1-of-a#{FILE_DELIMITER}  line1.2-of-a#{FILE_DELIMITER}  line1.3-of-a")
-      else
-        expect(e1_message).to eq("line1.1-of-a#{FILE_DELIMITER}  line1.2-of-a#{FILE_DELIMITER}  line1.3-of-a")
-        expect(e2_message).to eq("line1.1-of-z#{FILE_DELIMITER}  line1.2-of-z#{FILE_DELIMITER}  line1.3-of-z")
+      it "should only have one set of files open" do
+        subject.register
+        expect(lsof_proc.call).to eq("")
+        run_thread_proc.call
+        sleep 0.1
+        first_lsof = lsof_proc.call
+        expect(first_lsof).not_to eq("")
+        run_thread_proc.call
+        sleep 0.1
+        second_lsof = lsof_proc.call
+        expect(second_lsof).to eq(first_lsof)
       end
     end
   end

data/spec/spec_helper.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+
+require "logstash/devutils/rspec/spec_helper"
+
+class TracerBase
+  def initialize() @tracer = []; end
+
+  def trace_for(symbol)
+    params = @tracer.map {|k,v| k == symbol ? v : nil}.compact
+    params.empty? ? false : params
+  end
+
+  def clear()
+    @tracer.clear()
+  end
+end
+
+class FileLogTracer < TracerBase
+  def warn(*args) @tracer.push [:warn, args]; end
+  def error(*args) @tracer.push [:error, args]; end
+  def debug(*args) @tracer.push [:debug, args]; end
+  def info(*args) @tracer.push [:info, args]; end
+
+  def info?() true; end
+  def debug?() true; end
+  def warn?() true; end
+  def error?() true; end
+end
+
+class ComponentTracer < TracerBase
+  def accept(*args) @tracer.push [:accept, args]; end
+  def deliver(*args) @tracer.push [:deliver, args]; end
+end
+
+class CodecTracer < TracerBase
+  def decode_accept(ctx, data, listener)
+    @tracer.push [:decode_accept, [ctx, data]]
+    listener.process(ctx, {"message" => data})
+  end
+  def accept(listener)
+    @tracer.push [:accept, true]
+  end
+  def auto_flush()
+    @tracer.push [:auto_flush, true]
+  end
+  def close
+    @tracer.push [:close, true]
+  end
+  def clone
+    self.class.new
+  end
+end
+
+module Kernel
+  def pause_until(nap = 5, &block)
+    sq = SizedQueue.new(1)
+    th1 = Thread.new(sq) {|q| sleep nap; q.push(false) }
+    th2 = Thread.new(sq) do |q|
+      success = false
+      iters = nap * 5 + 1
+      iters.times do
+        break if !!(success = block.call)
+        sleep(0.2)
+      end
+      q.push(success)
+    end
+    sq.pop
+  end
+end
+
+RSpec::Matchers.define(:receive_call_and_args) do |m, args|
+  match do |actual|
+    actual.trace_for(m) == args
+   end
+
+  failure_message do
+    "Expecting method #{m} to receive: #{args} but got: #{actual.trace_for(m)}"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2015-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -16,18 +16,18 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.0.0.alpha0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
+        version: 2.0.0
     - - <
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 3.0.0.alpha0
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -64,18 +64,18 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.6.5
+        version: 0.7.0
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.7'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.6.5
+        version: 0.7.0
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0.7'
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -84,12 +84,12 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.3
+        version: 2.0.5
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.3
+        version: 2.0.5
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -149,6 +149,7 @@ files:
 - lib/logstash/inputs/file.rb
 - logstash-input-file.gemspec
 - spec/inputs/file_spec.rb
+- spec/spec_helper.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -171,9 +172,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Stream events from files.
 test_files:
 - spec/inputs/file_spec.rb
+- spec/spec_helper.rb