logstash-output-file 4.2.4 → 4.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b77d049c54896ab49beffdcfdc6c3d6263319550291b8e3e9bb7adbf693e132
-  data.tar.gz: 6c5149e2312badb7e33779600d0c65d43e0d1218ae0a7c0290f26878b706a71b
+  metadata.gz: a5b2636cc1cbd103ba360c3090ba03f8d1fa7c5b242f5295e6ce13b2e71f1914
+  data.tar.gz: 487b2d3fd9972f9e4f318766a08b170e2f575d27a296fdf1b9f57db4bff31400
 SHA512:
-  metadata.gz: 9040baa58fe70550583bdf6c745ca9f40a7317b660e961626ec1024f000850b3c8bb5d895d2ed12c95a85c66aadf889126c78dd9428bb35b63c7b591c3292678
-  data.tar.gz: d30d10c42110e1da662eda8a87ecc0e3d769502f2ea2e9b14f42e1cb3d82911f139f9e21c56a79ca2f9c2cae13722e09b8445931bc93e3b6dd0c12d0426fbebd
+  metadata.gz: 0bba901e47d963db64d1b1ee10b97c3743a194005a50a5449e5907953ff5661e4e4b9b6ac672a3d9788ea39934962be0bef46eaffc6951f52f242d735a660995
+  data.tar.gz: 7d599f979bddbc88bfa93cfcac7ed17917a528c5af9524d846fc9aead7a766fc8a3b968659e5e59d943562e684f9490509987fa9dd40dd575c487d1497b0b3b6

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 4.2.5
+  - Fix a bug introduced in v4.2.4 where events on low-volume pipelines could remain unflushed for long periods when `flush_interval` was non-zero [#70](https://github.com/logstash-plugins/logstash-output-file/pull/70)
+
 ## 4.2.4
-  - Fix a bug where flush interval was being called for each event when enabled #67
+  - Fix a bug where flush interval was being called for each event when enabled [#67](https://github.com/logstash-plugins/logstash-output-file/pull/67)
 
 ## 4.2.3
   - Docs: Set the default_codec doc attribute.

data/lib/logstash/outputs/file.rb

@@ -94,11 +94,12 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
     end
     @failure_path = File.join(@file_root, @filename_failure)
 
-
-    now = Time.now
-    @last_flush_cycle = now
-    @last_stale_cleanup_cycle = now
     @flush_interval = @flush_interval.to_i
+    if @flush_interval > 0
+      @flusher = Interval.start(@flush_interval, -> { flush_pending_files })
+    end
+
+    @last_stale_cleanup_cycle = Time.now
     @stale_cleanup_interval = 10
   end # def register
 
@@ -141,7 +142,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
           # append to the file
           chunks.each {|chunk| fd.write(chunk) }
         end
-        flush(fd)
+        fd.flush unless @flusher && @flusher.alive?
       end
 
       close_stale_files
@@ -150,6 +151,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
 
   public
   def close
+    @flusher.stop unless @flusher.nil?
     @io_mutex.synchronize do
       @logger.debug("Close: closing files")
 
@@ -200,27 +202,20 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
     parts.take_while { |part| part !~ FIELD_REF }.join(File::SEPARATOR)
   end
 
-  private
-  def flush(fd)
-    if flush_interval > 0
-      flush_pending_files
-    else
-      fd.flush
-    end
-  end
-
-  # every flush_interval seconds or so (triggered by events, but if there are no events there's no point flushing files anyway)
+  # the back-bone of @flusher, our periodic-flushing interval.
   private
   def flush_pending_files
-    return unless Time.now - @last_flush_cycle >= flush_interval
-    @logger.debug("Starting flush cycle")
+    @io_mutex.synchronize do
+      @logger.debug("Starting flush cycle")
 
-    @files.each do |path, fd|
-      @logger.debug("Flushing file", :path => path, :fd => fd)
-      fd.flush
+      @files.each do |path, fd|
+        @logger.debug("Flushing file", :path => path, :fd => fd)
+        fd.flush
+      end
     end
-
-    @last_flush_cycle = Time.now
+  rescue => e
+    # squash exceptions caught while flushing after logging them
+    @logger.error("Exception flushing files", :exception => e.message, :backtrace => e.backtrace)
   end
 
   # every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway)
@@ -295,6 +290,79 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
     end
     @files[path] = IOWriter.new(fd)
   end
+
+  ##
+  # Bare-bones utility for running a block of code at an interval.
+  #
+  class Interval
+    ##
+    # Initializes a new Interval with the given arguments and starts it before returning it.
+    #
+    # @param interval [Integer] (see: Interval#initialize)
+    # @param procsy [#call] (see: Interval#initialize)
+    #
+    # @return [Interval]
+    #
+    def self.start(interval, procsy)
+      self.new(interval, procsy).tap(&:start)
+    end
+
+    ##
+    # @param interval [Integer]: time in seconds to wait between calling the given proc
+    # @param procsy [#call]: proc or lambda to call periodically; must not raise exceptions.
+    def initialize(interval, procsy)
+      @interval = interval
+      @procsy = procsy
+
+      require 'thread' # Mutex, ConditionVariable, etc.
+      @mutex = Mutex.new
+      @sleeper = ConditionVariable.new
+    end
+
+    ##
+    # Starts the interval, or returns if it has already been started.
+    #
+    # @return [void]
+    def start
+      @mutex.synchronize do
+        return if @thread && @thread.alive?
+
+        @thread = Thread.new { run }
+      end
+    end
+
+    ##
+    # Stop the interval.
+    # Does not interrupt if execution is in-progress.
+    def stop
+      @mutex.synchronize do
+        @stopped = true
+      end
+
+      @thread && @thread.join
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      @thread && @thread.alive?
+    end
+
+    private
+
+    def run
+      @mutex.synchronize do
+        loop do
+          @sleeper.wait(@mutex, @interval)
+          break if @stopped
+
+          @procsy.call
+        end
+      end
+    ensure
+      @sleeper.broadcast
+    end
+  end # class LogStash::Outputs::File::Interval
 end # class LogStash::Outputs::File
 
 # wrapper class

data/logstash-output-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-file'
-  s.version         = '4.2.4'
+  s.version         = '4.2.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Writes events to files on disk"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/file_spec.rb

@@ -418,5 +418,50 @@ describe LogStash::Outputs::File do
         end
       end
     end
+
+    context "with non-zero flush interval" do
+      let(:temporary_output_file) { Stud::Temporary.pathname }
+
+      let(:event_count) { 10 }
+      let(:flush_interval) { 5 }
+
+      let(:events) do
+        event_count.times.map do |idx|
+          LogStash::Event.new("value" => idx)
+        end
+      end
+
+      let(:config) {
+        {
+            "path" => temporary_output_file,
+            "codec" => LogStash::Codecs::JSONLines.new,
+            "flush_interval" => flush_interval
+        }
+      }
+      let(:output) { LogStash::Outputs::File.new(config) }
+
+      before(:each) { output.register }
+      after(:each) do
+        output.close
+        File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
+      end
+
+      it 'eventually flushes without receiving additional events' do
+        output.multi_receive(events)
+
+        # events should not all be flushed just yet...
+        expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
+          contents && contents.lines.count < event_count
+        end
+
+        # wait for the flusher to run...
+        sleep(flush_interval + 1)
+
+        # events should all be flushed
+        expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
+          contents && contents.lines.count == event_count
+        end
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-file
 version: !ruby/object:Gem::Version
-  version: 4.2.4
+  version: 4.2.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-01 00:00:00.000000000 Z
+date: 2018-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement