logstash-input-file 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c56f2ba7106f61178bb34833bef500378352d39e
-  data.tar.gz: 8234370a6b5d313ff1057d5a6ea638cba656a83a
+  metadata.gz: 39708420cb297cae2205c5e2ef4e595f2b1652f0
+  data.tar.gz: 405f93ecc407455518f14fb9d16544e9357733b6
 SHA512:
-  metadata.gz: d64a4bd6c7610a1678244a016c7c97cceacf460f71b39a975d4a9954aabd33593bd8280a0833b3a5be57f94e0db1b23d4d2372a82ec611132a75f2aa4d415a7f
-  data.tar.gz: 4e86dc4b6b5c3c8fd70cf02e85e91ca02623199177fbeb29fd9be40821181a83f996d46076291e78a501e7c1ad0f7882471c474147c736368b588977129ce8f3
+  metadata.gz: c2460287157dd990831b5f91717f9004c9eb2af6469d73a15596ab029eb0579941c5a7d10a392e8e857cd02d2c84fe22e5b830d1d44ec7fba8789ba69e86bb32
+  data.tar.gz: 2fea4c08d6adcd180f920bf33083292d1d06a6bb9b209d67a5f150d4c6f800667ed230484df139f7b0bc0542fe2689b068e6b3f129981bbefcc8dfc312b22438

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.2
+ - Using filewatch >= 0.6.7, tail.quit closes files. Only one set of file
+ handles are used if run is called more than once.
+
 ## 1.0.1
  - Force dependency on filewatch >= 0.6.5 that fixes a sincedb bug
  - Better documentation and error handling regarding the "sincedb_path" parameter

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
-gemspec
+gemspec
+
+gem 'concurrent-ruby', '0.9.1'

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/file.rb

@@ -5,14 +5,62 @@ require "logstash/namespace"
 require "pathname"
 require "socket" # for Socket.gethostname
 
-# Stream events from files.
+# Stream events from files, normally by tailing them in a manner
+# similar to `tail -0F` but optionally reading them from the
+# beginning.
 #
 # By default, each event is assumed to be one line. If you would like
 # to join multiple log lines into one event, you'll want to use the
-# multiline codec.
+# multiline codec or filter.
 #
-# Files are followed in a manner similar to `tail -0F`. File rotation
-# is detected and handled by this input.
+# The plugin aims to track changing files and emit new content as it's
+# appended to each file. It's not well-suited for reading a file from
+# beginning to end and storing all of it in a single event (not even
+# with the multiline codec or filter).
+#
+# ==== Tracking of current position in watched files
+#
+# The plugin keeps track of the current position in each file by
+# recording it in a separate file named sincedb. This makes it
+# possible to stop and restart Logstash and have it pick up where it
+# left off without missing the lines that were added to the file while
+# Logstash was stopped.
+#
+# By default, the sincedb file is placed in the home directory of the
+# user running Logstash with a filename based on the filename patterns
+# being watched (i.e. the `path` option). Thus, changing the filename
+# patterns will result in a new sincedb file being used and any
+# existing current position state will be lost. If you change your
+# patterns with any frequency it might make sense to explicitly choose
+# a sincedb path with the `sincedb_path` option.
+#
+# Sincedb files are text files with four columns:
+#
+# . The inode number (or equivalent).
+# . The major device number of the file system (or equivalent).
+# . The minor device number of the file system (or equivalent).
+# . The current byte offset within the file.
+#
+# On non-Windows systems you can obtain the inode number of a file
+# with e.g. `ls -li`.
+#
+# ==== File rotation
+#
+# File rotation is detected and handled by this input, regardless of
+# whether the file is rotated via a rename or a copy operation. To
+# support programs that write to the rotated file for some time after
+# the rotation has taken place, include both the original filename and
+# the rotated filename (e.g. /var/log/syslog and /var/log/syslog.1) in
+# the filename patterns to watch (the `path` option). Note that the
+# rotated filename will be treated as a new file so if
+# `start_position` is set to 'beginning' the rotated file will be
+# reprocessed.
+#
+# With the default value of `start_position` ('end') any messages
+# written to the end of the file between the last read operation prior
+# to the rotation and its reopening under the new name (an interval
+# determined by the `stat_interval` and `discover_interval` options)
+# will not get picked up.
 class LogStash::Inputs::File < LogStash::Inputs::Base
   config_name "file"
 
@@ -21,15 +69,15 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   default :codec, "plain"
 
   # The path(s) to the file(s) to use as an input.
-  # You can use globs here, such as `/var/log/*.log`
+  # You can use filename patterns here, such as `/var/log/*.log`.
   # Paths must be absolute and cannot be relative.
   #
   # You may also configure multiple paths. See an example
   # on the <<array,Logstash configuration page>>.
   config :path, :validate => :array, :required => true
 
-  # Exclusions (matched against the filename, not full path). Globs
-  # are valid here, too. For example, if you have
+  # Exclusions (matched against the filename, not full path). Filename
+  # patterns are valid here, too. For example, if you have
   # [source,ruby]
   #     path => "/var/log/*"
   #
@@ -43,7 +91,8 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   # but increase the time to detect new log lines.
   config :stat_interval, :validate => :number, :default => 1
 
-  # How often (in seconds) we expand globs to discover new files to watch.
+  # How often (in seconds) we expand the filename patterns in the
+  # `path` option to discover new files to watch.
   config :discover_interval, :validate => :number, :default => 15
 
   # Path of the sincedb database file (keeps track of the current
@@ -59,11 +108,13 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   # Choose where Logstash starts initially reading files: at the beginning or
   # at the end. The default behavior treats files like live streams and thus
   # starts at the end. If you have old data you want to import, set this
-  # to 'beginning'
+  # to 'beginning'.
   #
-  # This option only modifies "first contact" situations where a file is new
-  # and not seen before. If a file has already been seen before, this option
-  # has no effect.
+  # This option only modifies "first contact" situations where a file
+  # is new and not seen before, i.e. files that don't have a current
+  # position recorded in a sincedb file read by Logstash. If a file
+  # has already been seen before, this option has no effect and the
+  # position recorded in the sincedb file will be used.
   config :start_position, :validate => [ "beginning", "end"], :default => "end"
 
   # set the new line delimiter, defaults to "\n"
@@ -132,12 +183,15 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
     end
   end # def register
 
-  public
-  def run(queue)
+  def begin_tailing
+    teardown   # if the pipeline restarts this input.
     @tail = FileWatch::Tail.new(@tail_config)
     @tail.logger = @logger
     @path.each { |path| @tail.tail(path) }
+  end
 
+  def run(queue)
+    begin_tailing
     @tail.subscribe do |path, line|
       @logger.debug? && @logger.debug("Received line", :path => path, :text => line)
       @codec.decode(line) do |event|
@@ -153,10 +207,9 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
 
   public
   def teardown
-    if @tail
-      @tail.sincedb_write
-      @tail.quit
-      @tail = nil
-    end
+    # in filewatch >= 0.6.7, quit will close and forget all files
+    # but it will write their last read positions to since_db
+    # beforehand
+    @tail.quit if @tail
   end # def teardown
 end # class LogStash::Inputs::File

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '1.0.1'
+  s.version         = '1.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,14 +20,14 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0.alpha0'
 
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'addressable'
-  s.add_runtime_dependency 'filewatch', ['>= 0.6.5', '~> 0.6']
+  s.add_runtime_dependency 'filewatch', ['>= 0.6.7', '~> 0.6']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
-  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-devutils', '= 0.0.15'
   s.add_development_dependency 'logstash-codec-json'
 end
 

data/spec/inputs/file_spec.rb

@@ -6,6 +6,14 @@ require "stud/temporary"
 require "logstash/inputs/file"
 
 describe "inputs/file" do
+  before(:all) do
+    @abort_on_exception = Thread.abort_on_exception
+    Thread.abort_on_exception = true
+  end
+
+  after(:all) do
+    Thread.abort_on_exception = @abort_on_exception
+  end
 
   delimiter = (LogStash::Environment.windows? ? "\r\n" : "\n")
 
@@ -180,4 +188,45 @@ describe "inputs/file" do
       expect { subject.register }.to raise_error(ArgumentError)
     end
   end
+
+  context "when #run is called multiple times", :unix => true do
+    let(:tmpdir_path)  { Stud::Temporary.directory }
+    let(:sincedb_path) { Stud::Temporary.pathname }
+    let(:file_path)    { "#{tmpdir_path}/a.log" }
+    let(:buffer)       { [] }
+    let(:lsof)         { [] }
+    let(:stop_proc) do
+      lambda do |input, arr|
+        Thread.new(input, arr) do |i, a|
+          sleep 0.5
+          a << `lsof -p #{Process.pid} | grep "a.log"`
+          i.teardown
+        end
+      end
+    end
+
+    subject { LogStash::Inputs::File.new("path" => tmpdir_path + "/*.log", "start_position" => "beginning", "sincedb_path" => sincedb_path) }
+
+    after :each do
+      FileUtils.rm_rf(tmpdir_path)
+      FileUtils.rm_rf(sincedb_path)
+    end
+    before do
+      File.open(file_path, "w") do |fd|
+        fd.puts('foo')
+        fd.puts('bar')
+      end
+    end
+    it "should only have one set of files open" do
+      subject.register
+      lsof_before = `lsof -p #{Process.pid} | grep #{file_path}`
+      expect(lsof_before).to eq("")
+      stop_proc.call(subject, lsof)
+      subject.run(buffer)
+      expect(lsof.first).not_to eq("")
+      stop_proc.call(subject, lsof)
+      subject.run(buffer)
+      expect(lsof.last).to eq(lsof.first)
+    end
+  end
 end

metadata

@@ -1,25 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-12 00:00:00.000000000 Z
+date: 2015-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -27,99 +18,108 @@ dependencies:
         version: 1.4.0
     - - <
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.0.alpha0
+  name: logstash-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0.alpha0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: addressable
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: addressable
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: filewatch
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.6.5
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.6'
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.6.5
+        version: 0.6.7
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.6'
+  name: filewatch
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.7
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.19
+        version: '0.6'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.19
+  name: stud
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.19
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.15
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.15
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-json
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -159,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 4
 summary: Stream events from files.