logstash-input-file 2.1.3 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6df644dbc8e574f5611d6c27cef8eb372ecae09d
-  data.tar.gz: ca7e69428222362a80af560b1728bdc97abf6008
+  metadata.gz: 6412fcb09af0a0e32879a180c77e409c868caa43
+  data.tar.gz: 1d2fe6993ca73c122b2002a6c72c5d0f68a1fb9a
 SHA512:
-  metadata.gz: e60e2247d23a0dad2f69fff24aafb80fb04f44b10a33fc1bb332a19c02a6a3930c2baebf597521646c40a3aefb5e77cee6541e23737e6b36fab863dc42dcfeed
-  data.tar.gz: 9b1f5ee7af2a8b3340be9d11592cc1d1056198c07ce7deec5b204dd91c98fb534903d50aae27d5a52effef4ec9fbfaaee90798186d7434fc8ab9139f2c8e4543
+  metadata.gz: 2418a21eb0d3ea85185158c69ccbb971b332da29476fcdd7508249f3c28edda61c6a98e6886484a4a9bda6fd3e2b9ac0057eb6a8a84de17b2af075f4671cbae6
+  data.tar.gz: 189d29b3ccfbabdc6ad77d510cccb416dc0c738564676101a76beec9f37129c922f79ba613fb280d7486e70b4450693f9f08872a50372f93e467b09640cd9e73

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.2.0
+ - Use ruby-filewatch 0.8.0, major rework of filewatch. See [Pull Request 74](https://github.com/jordansissel/ruby-filewatch/pull/74)
+ - add max_open_files config option, defaults to 4095, the input will process much more than this but have this number of files open at any time - files are closed based on the close_older setting, thereby making others openable.
+ - Changes the close_older logic to measure the time since the file was last read internlly rather than using the file stat modified time.
+ - Use logstash-codec-multiline 2.0.7, fixes a bug with auto_flush deadlocking when multiple file inputs are defined in the LS config.
+
 ## 2.1.3
  - Use ruby-filewatch 0.7.1, re-enable close after file is modified again
 

data/lib/logstash/inputs/file.rb

@@ -35,6 +35,10 @@ require "socket" # for Socket.gethostname
 # patterns with any frequency it might make sense to explicitly choose
 # a sincedb path with the `sincedb_path` option.
 #
+# A different `sincedb_path` must be used for each input. Using the same
+# path will cause issues. The read checkpoints for each input must be
+# stored in a different path so the information does not override.
+#
 # Sincedb files are text files with four columns:
 #
 # . The inode number (or equivalent).
@@ -141,11 +145,24 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
   # longer ignored and any new data is read. The default is 24 hours.
   config :ignore_older, :validate => :number, :default => 24 * 60 * 60
 
-  # If this option is specified, the file input closes any files that remain
-  # unmodified for longer than the specified timespan in seconds.
+  # If this option is specified, the file input closes any files that were last
+  # read the specified timespan in seconds ago.
+  # This has different implications depending on if a file is being tailed or
+  # read. If tailing, and there is a large time gap in incoming data the file
+  # can be closed (allowing other files to be opened) but will be queued for
+  # reopening when new data is detected. If reading, the file will be closed
+  # after closed_older seconds from when the last bytes were read.
   # The default is 1 hour
   config :close_older, :validate => :number, :default => 1 * 60 * 60
 
+  # What is the maximum number of file_handles that this input consumes
+  # at any one time. Use close_older to close some files if you need to
+  # process more files than this number. This should not be set to the
+  # maximum the OS can do because file handles are needed for other
+  # LS plugins and OS processes.
+  # The default of 4095 is set in filewatch.
+  config :max_open_files, :validate => :number
+
   public
   def register
     require "addressable/uri"
@@ -161,7 +178,8 @@ class LogStash::Inputs::File < LogStash::Inputs::Base
       :sincedb_write_interval => @sincedb_write_interval,
       :delimiter => @delimiter,
       :ignore_older => @ignore_older,
-      :close_older => @close_older
+      :close_older => @close_older,
+      :max_open_files => @max_open_files
     }
 
     @path.each do |path|

data/logstash-input-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-file'
-  s.version         = '2.1.3'
+  s.version         = '2.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -24,8 +24,8 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'addressable'
-  s.add_runtime_dependency 'filewatch', ['>= 0.7.1', '~> 0.7']
-  s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.5']
+  s.add_runtime_dependency 'filewatch', ['>= 0.8.0', '~> 0.8']
+  s.add_runtime_dependency 'logstash-codec-multiline', ['~> 2.0.7']
 
   s.add_development_dependency 'stud', ['~> 0.0.19']
   s.add_development_dependency 'logstash-devutils'

data/spec/inputs/file_spec.rb

@@ -152,6 +152,10 @@ describe LogStash::Inputs::File do
     let(:sincedb_path) { Stud::Temporary.pathname }
     let(:tmpdir_path)  { Stud::Temporary.directory }
 
+    after :each do
+      FileUtils.rm_rf(sincedb_path)
+    end
+
     context "when data exists and then more data is appended" do
       subject { described_class.new(conf) }
 
@@ -369,6 +373,7 @@ describe LogStash::Inputs::File do
         conf.update(
           "path" => tmpdir_path + "/*.log",
           "start_position" => "beginning",
+          "stat_interval" => 0.1,
           "sincedb_path" => sincedb_path)
 
         File.open(file_path, "w") do |fd|
@@ -382,14 +387,98 @@ describe LogStash::Inputs::File do
         subject.register
         expect(lsof_proc.call).to eq("")
         run_thread_proc.call
-        sleep 0.1
+        sleep 0.25
         first_lsof = lsof_proc.call
-        expect(first_lsof).not_to eq("")
+        expect(first_lsof.scan(file_path).size).to eq(1)
         run_thread_proc.call
-        sleep 0.1
+        sleep 0.25
         second_lsof = lsof_proc.call
-        expect(second_lsof).to eq(first_lsof)
+        expect(second_lsof.scan(file_path).size).to eq(1)
+      end
+    end
+
+    describe "specifying max_open_files" do
+      subject { described_class.new(conf) }
+      before do
+        File.open("#{tmpdir_path}/a.log", "w") do |fd|
+          fd.puts("line1-of-a")
+          fd.puts("line2-of-a")
+          fd.fsync
+        end
+        File.open("#{tmpdir_path}/z.log", "w") do |fd|
+          fd.puts("line1-of-z")
+          fd.puts("line2-of-z")
+          fd.fsync
+        end
       end
+
+      context "when close_older is NOT specified" do
+        before do
+          conf.clear
+          conf.update(
+                "type" => "blah",
+                "path" => "#{tmpdir_path}/*.log",
+                "sincedb_path" => sincedb_path,
+                "stat_interval" => 0.1,
+                "max_open_files" => 1,
+                "start_position" => "beginning",
+                "delimiter" => FILE_DELIMITER)
+          subject.register
+          Thread.new { subject.run(events) }
+          sleep 0.1
+        end
+        it "collects line events from only one file" do
+          # wait for one path to be mapped as identity
+          expect(pause_until{ subject.codec.identity_count == 1 }).to be_truthy
+          subject.stop
+          # stop flushes last event
+          expect(pause_until{ events.size == 2 }).to be_truthy
+
+          e1, e2 = events
+          if Dir.glob("#{tmpdir_path}/*.log").first =~ %r{a\.log}
+            #linux and OSX have different retrieval order
+            expect(e1["message"]).to eq("line1-of-a")
+            expect(e2["message"]).to eq("line2-of-a")
+          else
+            expect(e1["message"]).to eq("line1-of-z")
+            expect(e2["message"]).to eq("line2-of-z")
+          end
+        end
+      end
+
+      context "when close_older IS specified" do
+        before do
+          conf.update(
+                "type" => "blah",
+                "path" => "#{tmpdir_path}/*.log",
+                "sincedb_path" => sincedb_path,
+                "stat_interval" => 0.1,
+                "max_open_files" => 1,
+                "close_older" => 1,
+                "start_position" => "beginning",
+                "delimiter" => FILE_DELIMITER)
+          subject.register
+          Thread.new { subject.run(events) }
+          sleep 0.1
+        end
+
+        it "collects line events from both files" do
+          # close flushes last event of each identity
+          expect(pause_until{ events.size == 4 }).to be_truthy
+          subject.stop
+          if Dir.glob("#{tmpdir_path}/*.log").first =~ %r{a\.log}
+            #linux and OSX have different retrieval order
+            e1, e2, e3, e4 = events
+          else
+            e3, e4, e1, e2 = events
+          end
+          expect(e1["message"]).to eq("line1-of-a")
+          expect(e2["message"]).to eq("line2-of-a")
+          expect(e3["message"]).to eq("line1-of-z")
+          expect(e4["message"]).to eq("line2-of-z")
+        end
+      end
+
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-file
 version: !ruby/object:Gem::Version
-  version: 2.1.3
+  version: 2.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-04 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -64,18 +64,18 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.0
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.8.0
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0.8'
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -84,12 +84,12 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.5
+        version: 2.0.7
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.5
+        version: 2.0.7
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
@@ -147,7 +147,6 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/inputs/file.rb
-- lib/logstash/inputs/identity_map_codec_component.rb
 - logstash-input-file.gemspec
 - spec/inputs/file_spec.rb
 - spec/spec_helper.rb

data/lib/logstash/inputs/identity_map_codec_component.rb

@@ -1,56 +0,0 @@
-# encoding: utf-8
-require "logstash/inputs/component"
-require "logstash/codecs/identity_map_codec"
-
-module LogStash module Inputs class IdentityMapCodecComponent
-  include Component
-
-  attr_reader :codec
-
-  def add_codec(codec)
-    @codec = LogStash::Codecs::IdentityMapCodec.new(codec)
-    self
-  end
-
-  def stop
-    @codec.close
-  end
-
-  def do_work(context, data)
-    do_line(context, data) || do_eviction(context, data)
-  end
-
-  def process(context, data)
-    # data should be an event
-    deliver(context, data)
-  end
-
-  private
-
-  def do_line(context, data)
-    return false unless line?(context)
-    @codec.decode_accept(context, data, self)
-    # above should call back on #process
-    true
-  end
-
-  def do_eviction(context, data)
-    return false unless evicting?(context)
-    path = context[:path]
-    @codec.evict(path) if path
-    true
-  end
-
-  def line?(ctx)
-    action(ctx) == "line"
-  end
-
-  def evicting?(ctx)
-    _action = action(ctx)
-    _action == "timed_out" || _action == "deleted"
-  end
-
-  def action(ctx)
-    ctx[:action]
-  end
-end end end