logstash-output-file 4.2.4 → 4.2.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/lib/logstash/outputs/file.rb +90 -22
- data/logstash-output-file.gemspec +1 -1
- data/spec/outputs/file_spec.rb +45 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a5b2636cc1cbd103ba360c3090ba03f8d1fa7c5b242f5295e6ce13b2e71f1914
|
4
|
+
data.tar.gz: 487b2d3fd9972f9e4f318766a08b170e2f575d27a296fdf1b9f57db4bff31400
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0bba901e47d963db64d1b1ee10b97c3743a194005a50a5449e5907953ff5661e4e4b9b6ac672a3d9788ea39934962be0bef46eaffc6951f52f242d735a660995
|
7
|
+
data.tar.gz: 7d599f979bddbc88bfa93cfcac7ed17917a528c5af9524d846fc9aead7a766fc8a3b968659e5e59d943562e684f9490509987fa9dd40dd575c487d1497b0b3b6
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
+
## 4.2.5
|
2
|
+
- Fix a bug introduced in v4.2.4 where events on low-volume pipelines could remain unflushed for long periods when `flush_interval` was non-zero [#70](https://github.com/logstash-plugins/logstash-output-file/pull/70)
|
3
|
+
|
1
4
|
## 4.2.4
|
2
|
-
- Fix a bug where flush interval was being called for each event when enabled #67
|
5
|
+
- Fix a bug where flush interval was being called for each event when enabled [#67](https://github.com/logstash-plugins/logstash-output-file/pull/67)
|
3
6
|
|
4
7
|
## 4.2.3
|
5
8
|
- Docs: Set the default_codec doc attribute.
|
@@ -94,11 +94,12 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
|
|
94
94
|
end
|
95
95
|
@failure_path = File.join(@file_root, @filename_failure)
|
96
96
|
|
97
|
-
|
98
|
-
now = Time.now
|
99
|
-
@last_flush_cycle = now
|
100
|
-
@last_stale_cleanup_cycle = now
|
101
97
|
@flush_interval = @flush_interval.to_i
|
98
|
+
if @flush_interval > 0
|
99
|
+
@flusher = Interval.start(@flush_interval, -> { flush_pending_files })
|
100
|
+
end
|
101
|
+
|
102
|
+
@last_stale_cleanup_cycle = Time.now
|
102
103
|
@stale_cleanup_interval = 10
|
103
104
|
end # def register
|
104
105
|
|
@@ -141,7 +142,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
|
|
141
142
|
# append to the file
|
142
143
|
chunks.each {|chunk| fd.write(chunk) }
|
143
144
|
end
|
144
|
-
flush
|
145
|
+
fd.flush unless @flusher && @flusher.alive?
|
145
146
|
end
|
146
147
|
|
147
148
|
close_stale_files
|
@@ -150,6 +151,7 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
|
|
150
151
|
|
151
152
|
public
|
152
153
|
def close
|
154
|
+
@flusher.stop unless @flusher.nil?
|
153
155
|
@io_mutex.synchronize do
|
154
156
|
@logger.debug("Close: closing files")
|
155
157
|
|
@@ -200,27 +202,20 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
|
|
200
202
|
parts.take_while { |part| part !~ FIELD_REF }.join(File::SEPARATOR)
|
201
203
|
end
|
202
204
|
|
203
|
-
|
204
|
-
def flush(fd)
|
205
|
-
if flush_interval > 0
|
206
|
-
flush_pending_files
|
207
|
-
else
|
208
|
-
fd.flush
|
209
|
-
end
|
210
|
-
end
|
211
|
-
|
212
|
-
# every flush_interval seconds or so (triggered by events, but if there are no events there's no point flushing files anyway)
|
205
|
+
# the back-bone of @flusher, our periodic-flushing interval.
|
213
206
|
private
|
214
207
|
def flush_pending_files
|
215
|
-
|
216
|
-
|
208
|
+
@io_mutex.synchronize do
|
209
|
+
@logger.debug("Starting flush cycle")
|
217
210
|
|
218
|
-
|
219
|
-
|
220
|
-
|
211
|
+
@files.each do |path, fd|
|
212
|
+
@logger.debug("Flushing file", :path => path, :fd => fd)
|
213
|
+
fd.flush
|
214
|
+
end
|
221
215
|
end
|
222
|
-
|
223
|
-
|
216
|
+
rescue => e
|
217
|
+
# squash exceptions caught while flushing after logging them
|
218
|
+
@logger.error("Exception flushing files", :exception => e.message, :backtrace => e.backtrace)
|
224
219
|
end
|
225
220
|
|
226
221
|
# every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway)
|
@@ -295,6 +290,79 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
|
|
295
290
|
end
|
296
291
|
@files[path] = IOWriter.new(fd)
|
297
292
|
end
|
293
|
+
|
294
|
+
##
|
295
|
+
# Bare-bones utility for running a block of code at an interval.
|
296
|
+
#
|
297
|
+
class Interval
|
298
|
+
##
|
299
|
+
# Initializes a new Interval with the given arguments and starts it before returning it.
|
300
|
+
#
|
301
|
+
# @param interval [Integer] (see: Interval#initialize)
|
302
|
+
# @param procsy [#call] (see: Interval#initialize)
|
303
|
+
#
|
304
|
+
# @return [Interval]
|
305
|
+
#
|
306
|
+
def self.start(interval, procsy)
|
307
|
+
self.new(interval, procsy).tap(&:start)
|
308
|
+
end
|
309
|
+
|
310
|
+
##
|
311
|
+
# @param interval [Integer]: time in seconds to wait between calling the given proc
|
312
|
+
# @param procsy [#call]: proc or lambda to call periodically; must not raise exceptions.
|
313
|
+
def initialize(interval, procsy)
|
314
|
+
@interval = interval
|
315
|
+
@procsy = procsy
|
316
|
+
|
317
|
+
require 'thread' # Mutex, ConditionVariable, etc.
|
318
|
+
@mutex = Mutex.new
|
319
|
+
@sleeper = ConditionVariable.new
|
320
|
+
end
|
321
|
+
|
322
|
+
##
|
323
|
+
# Starts the interval, or returns if it has already been started.
|
324
|
+
#
|
325
|
+
# @return [void]
|
326
|
+
def start
|
327
|
+
@mutex.synchronize do
|
328
|
+
return if @thread && @thread.alive?
|
329
|
+
|
330
|
+
@thread = Thread.new { run }
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
##
|
335
|
+
# Stop the interval.
|
336
|
+
# Does not interrupt if execution is in-progress.
|
337
|
+
def stop
|
338
|
+
@mutex.synchronize do
|
339
|
+
@stopped = true
|
340
|
+
end
|
341
|
+
|
342
|
+
@thread && @thread.join
|
343
|
+
end
|
344
|
+
|
345
|
+
##
|
346
|
+
# @return [Boolean]
|
347
|
+
def alive?
|
348
|
+
@thread && @thread.alive?
|
349
|
+
end
|
350
|
+
|
351
|
+
private
|
352
|
+
|
353
|
+
def run
|
354
|
+
@mutex.synchronize do
|
355
|
+
loop do
|
356
|
+
@sleeper.wait(@mutex, @interval)
|
357
|
+
break if @stopped
|
358
|
+
|
359
|
+
@procsy.call
|
360
|
+
end
|
361
|
+
end
|
362
|
+
ensure
|
363
|
+
@sleeper.broadcast
|
364
|
+
end
|
365
|
+
end # class LogStash::Outputs::File::Interval
|
298
366
|
end # class LogStash::Outputs::File
|
299
367
|
|
300
368
|
# wrapper class
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-file'
|
4
|
-
s.version = '4.2.
|
4
|
+
s.version = '4.2.5'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Writes events to files on disk"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/outputs/file_spec.rb
CHANGED
@@ -418,5 +418,50 @@ describe LogStash::Outputs::File do
|
|
418
418
|
end
|
419
419
|
end
|
420
420
|
end
|
421
|
+
|
422
|
+
context "with non-zero flush interval" do
|
423
|
+
let(:temporary_output_file) { Stud::Temporary.pathname }
|
424
|
+
|
425
|
+
let(:event_count) { 10 }
|
426
|
+
let(:flush_interval) { 5 }
|
427
|
+
|
428
|
+
let(:events) do
|
429
|
+
event_count.times.map do |idx|
|
430
|
+
LogStash::Event.new("value" => idx)
|
431
|
+
end
|
432
|
+
end
|
433
|
+
|
434
|
+
let(:config) {
|
435
|
+
{
|
436
|
+
"path" => temporary_output_file,
|
437
|
+
"codec" => LogStash::Codecs::JSONLines.new,
|
438
|
+
"flush_interval" => flush_interval
|
439
|
+
}
|
440
|
+
}
|
441
|
+
let(:output) { LogStash::Outputs::File.new(config) }
|
442
|
+
|
443
|
+
before(:each) { output.register }
|
444
|
+
after(:each) do
|
445
|
+
output.close
|
446
|
+
File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
|
447
|
+
end
|
448
|
+
|
449
|
+
it 'eventually flushes without receiving additional events' do
|
450
|
+
output.multi_receive(events)
|
451
|
+
|
452
|
+
# events should not all be flushed just yet...
|
453
|
+
expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
|
454
|
+
contents && contents.lines.count < event_count
|
455
|
+
end
|
456
|
+
|
457
|
+
# wait for the flusher to run...
|
458
|
+
sleep(flush_interval + 1)
|
459
|
+
|
460
|
+
# events should all be flushed
|
461
|
+
expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
|
462
|
+
contents && contents.lines.count == event_count
|
463
|
+
end
|
464
|
+
end
|
465
|
+
end
|
421
466
|
end
|
422
467
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-file
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.2.
|
4
|
+
version: 4.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-06-
|
11
|
+
date: 2018-06-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|