logstash-output-file 4.2.4 → 4.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8b77d049c54896ab49beffdcfdc6c3d6263319550291b8e3e9bb7adbf693e132
4
- data.tar.gz: 6c5149e2312badb7e33779600d0c65d43e0d1218ae0a7c0290f26878b706a71b
3
+ metadata.gz: a5b2636cc1cbd103ba360c3090ba03f8d1fa7c5b242f5295e6ce13b2e71f1914
4
+ data.tar.gz: 487b2d3fd9972f9e4f318766a08b170e2f575d27a296fdf1b9f57db4bff31400
5
5
  SHA512:
6
- metadata.gz: 9040baa58fe70550583bdf6c745ca9f40a7317b660e961626ec1024f000850b3c8bb5d895d2ed12c95a85c66aadf889126c78dd9428bb35b63c7b591c3292678
7
- data.tar.gz: d30d10c42110e1da662eda8a87ecc0e3d769502f2ea2e9b14f42e1cb3d82911f139f9e21c56a79ca2f9c2cae13722e09b8445931bc93e3b6dd0c12d0426fbebd
6
+ metadata.gz: 0bba901e47d963db64d1b1ee10b97c3743a194005a50a5449e5907953ff5661e4e4b9b6ac672a3d9788ea39934962be0bef46eaffc6951f52f242d735a660995
7
+ data.tar.gz: 7d599f979bddbc88bfa93cfcac7ed17917a528c5af9524d846fc9aead7a766fc8a3b968659e5e59d943562e684f9490509987fa9dd40dd575c487d1497b0b3b6
@@ -1,5 +1,8 @@
1
+ ## 4.2.5
2
+ - Fix a bug introduced in v4.2.4 where events on low-volume pipelines could remain unflushed for long periods when `flush_interval` was non-zero [#70](https://github.com/logstash-plugins/logstash-output-file/pull/70)
3
+
1
4
  ## 4.2.4
2
- - Fix a bug where flush interval was being called for each event when enabled #67
5
+ - Fix a bug where flush interval was being called for each event when enabled [#67](https://github.com/logstash-plugins/logstash-output-file/pull/67)
3
6
 
4
7
  ## 4.2.3
5
8
  - Docs: Set the default_codec doc attribute.
@@ -94,11 +94,12 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
94
94
  end
95
95
  @failure_path = File.join(@file_root, @filename_failure)
96
96
 
97
-
98
- now = Time.now
99
- @last_flush_cycle = now
100
- @last_stale_cleanup_cycle = now
101
97
  @flush_interval = @flush_interval.to_i
98
+ if @flush_interval > 0
99
+ @flusher = Interval.start(@flush_interval, -> { flush_pending_files })
100
+ end
101
+
102
+ @last_stale_cleanup_cycle = Time.now
102
103
  @stale_cleanup_interval = 10
103
104
  end # def register
104
105
 
@@ -141,7 +142,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
141
142
  # append to the file
142
143
  chunks.each {|chunk| fd.write(chunk) }
143
144
  end
144
- flush(fd)
145
+ fd.flush unless @flusher && @flusher.alive?
145
146
  end
146
147
 
147
148
  close_stale_files
@@ -150,6 +151,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
150
151
 
151
152
  public
152
153
  def close
154
+ @flusher.stop unless @flusher.nil?
153
155
  @io_mutex.synchronize do
154
156
  @logger.debug("Close: closing files")
155
157
 
@@ -200,27 +202,20 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
200
202
  parts.take_while { |part| part !~ FIELD_REF }.join(File::SEPARATOR)
201
203
  end
202
204
 
203
- private
204
- def flush(fd)
205
- if flush_interval > 0
206
- flush_pending_files
207
- else
208
- fd.flush
209
- end
210
- end
211
-
212
- # every flush_interval seconds or so (triggered by events, but if there are no events there's no point flushing files anyway)
205
+ # the back-bone of @flusher, our periodic-flushing interval.
213
206
  private
214
207
  def flush_pending_files
215
- return unless Time.now - @last_flush_cycle >= flush_interval
216
- @logger.debug("Starting flush cycle")
208
+ @io_mutex.synchronize do
209
+ @logger.debug("Starting flush cycle")
217
210
 
218
- @files.each do |path, fd|
219
- @logger.debug("Flushing file", :path => path, :fd => fd)
220
- fd.flush
211
+ @files.each do |path, fd|
212
+ @logger.debug("Flushing file", :path => path, :fd => fd)
213
+ fd.flush
214
+ end
221
215
  end
222
-
223
- @last_flush_cycle = Time.now
216
+ rescue => e
217
+ # squash exceptions caught while flushing after logging them
218
+ @logger.error("Exception flushing files", :exception => e.message, :backtrace => e.backtrace)
224
219
  end
225
220
 
226
221
  # every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway)
@@ -295,6 +290,79 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
295
290
  end
296
291
  @files[path] = IOWriter.new(fd)
297
292
  end
293
+
294
+ ##
295
+ # Bare-bones utility for running a block of code at an interval.
296
+ #
297
+ class Interval
298
+ ##
299
+ # Initializes a new Interval with the given arguments and starts it before returning it.
300
+ #
301
+ # @param interval [Integer] (see: Interval#initialize)
302
+ # @param procsy [#call] (see: Interval#initialize)
303
+ #
304
+ # @return [Interval]
305
+ #
306
+ def self.start(interval, procsy)
307
+ self.new(interval, procsy).tap(&:start)
308
+ end
309
+
310
+ ##
311
+ # @param interval [Integer]: time in seconds to wait between calling the given proc
312
+ # @param procsy [#call]: proc or lambda to call periodically; must not raise exceptions.
313
+ def initialize(interval, procsy)
314
+ @interval = interval
315
+ @procsy = procsy
316
+
317
+ require 'thread' # Mutex, ConditionVariable, etc.
318
+ @mutex = Mutex.new
319
+ @sleeper = ConditionVariable.new
320
+ end
321
+
322
+ ##
323
+ # Starts the interval, or returns if it has already been started.
324
+ #
325
+ # @return [void]
326
+ def start
327
+ @mutex.synchronize do
328
+ return if @thread && @thread.alive?
329
+
330
+ @thread = Thread.new { run }
331
+ end
332
+ end
333
+
334
+ ##
335
+ # Stop the interval.
336
+ # Does not interrupt if execution is in-progress.
337
+ def stop
338
+ @mutex.synchronize do
339
+ @stopped = true
340
+ end
341
+
342
+ @thread && @thread.join
343
+ end
344
+
345
+ ##
346
+ # @return [Boolean]
347
+ def alive?
348
+ @thread && @thread.alive?
349
+ end
350
+
351
+ private
352
+
353
+ def run
354
+ @mutex.synchronize do
355
+ loop do
356
+ @sleeper.wait(@mutex, @interval)
357
+ break if @stopped
358
+
359
+ @procsy.call
360
+ end
361
+ end
362
+ ensure
363
+ @sleeper.broadcast
364
+ end
365
+ end # class LogStash::Outputs::File::Interval
298
366
  end # class LogStash::Outputs::File
299
367
 
300
368
  # wrapper class
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-file'
4
- s.version = '4.2.4'
4
+ s.version = '4.2.5'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Writes events to files on disk"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -418,5 +418,50 @@ describe LogStash::Outputs::File do
418
418
  end
419
419
  end
420
420
  end
421
+
422
+ context "with non-zero flush interval" do
423
+ let(:temporary_output_file) { Stud::Temporary.pathname }
424
+
425
+ let(:event_count) { 10 }
426
+ let(:flush_interval) { 5 }
427
+
428
+ let(:events) do
429
+ event_count.times.map do |idx|
430
+ LogStash::Event.new("value" => idx)
431
+ end
432
+ end
433
+
434
+ let(:config) {
435
+ {
436
+ "path" => temporary_output_file,
437
+ "codec" => LogStash::Codecs::JSONLines.new,
438
+ "flush_interval" => flush_interval
439
+ }
440
+ }
441
+ let(:output) { LogStash::Outputs::File.new(config) }
442
+
443
+ before(:each) { output.register }
444
+ after(:each) do
445
+ output.close
446
+ File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
447
+ end
448
+
449
+ it 'eventually flushes without receiving additional events' do
450
+ output.multi_receive(events)
451
+
452
+ # events should not all be flushed just yet...
453
+ expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
454
+ contents && contents.lines.count < event_count
455
+ end
456
+
457
+ # wait for the flusher to run...
458
+ sleep(flush_interval + 1)
459
+
460
+ # events should all be flushed
461
+ expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
462
+ contents && contents.lines.count == event_count
463
+ end
464
+ end
465
+ end
421
466
  end
422
467
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-file
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.2.4
4
+ version: 4.2.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-01 00:00:00.000000000 Z
11
+ date: 2018-06-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement