fluentd 1.9.2-x64-mingw32 → 1.9.3-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -1
  3. data/CHANGELOG.md +37 -0
  4. data/lib/fluent/env.rb +2 -0
  5. data/lib/fluent/plugin/buf_file.rb +12 -4
  6. data/lib/fluent/plugin/buf_file_single.rb +1 -2
  7. data/lib/fluent/plugin/buffer.rb +19 -3
  8. data/lib/fluent/plugin/buffer/chunk.rb +1 -1
  9. data/lib/fluent/plugin/buffer/file_chunk.rb +3 -3
  10. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  11. data/lib/fluent/plugin/in_tail.rb +162 -131
  12. data/lib/fluent/plugin/in_unix.rb +0 -6
  13. data/lib/fluent/plugin/out_file.rb +6 -28
  14. data/lib/fluent/plugin/out_secondary_file.rb +2 -4
  15. data/lib/fluent/plugin/output.rb +1 -1
  16. data/lib/fluent/plugin/parser_multiline.rb +48 -2
  17. data/lib/fluent/plugin_helper/server.rb +5 -1
  18. data/lib/fluent/plugin_id.rb +1 -1
  19. data/lib/fluent/supervisor.rb +11 -3
  20. data/lib/fluent/version.rb +1 -1
  21. data/test/command/test_fluentd.rb +31 -2
  22. data/test/plugin/test_buf_file.rb +84 -4
  23. data/test/plugin/test_buf_file_single.rb +2 -2
  24. data/test/plugin/test_buffer.rb +18 -2
  25. data/test/plugin/test_buffer_file_chunk.rb +13 -12
  26. data/test/plugin/test_buffer_file_single_chunk.rb +1 -1
  27. data/test/plugin/test_in_tail.rb +82 -29
  28. data/test/plugin/test_out_copy.rb +3 -0
  29. data/test/plugin/test_output_as_buffered_backup.rb +48 -0
  30. data/test/plugin/test_parser_multiline.rb +11 -0
  31. data/test/plugin_helper/data/cert/generate_cert.rb +2 -2
  32. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +26 -26
  33. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +25 -25
  34. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +18 -18
  35. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +18 -18
  36. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +26 -26
  37. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +25 -25
  38. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +19 -19
  39. data/test/plugin_helper/data/cert/with_ca/cert.pem +18 -18
  40. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +26 -26
  41. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +25 -25
  42. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +17 -17
  43. data/test/plugin_helper/data/cert/without_ca/cert.pem +17 -17
  44. data/test/plugin_helper/test_http_server_helper.rb +1 -9
  45. data/test/test_logger_initializer.rb +20 -0
  46. data/test/test_plugin_id.rb +18 -0
  47. data/test/test_supervisor.rb +1 -1
  48. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 976b5918a9761704f4c808761a5988c6f95f1b86e6fd7f7fa68f68f5de4b64ce
4
- data.tar.gz: 37a6608a19b50fc7d4815df34fb9321edfd48547d32513a0c631d297580ca71b
3
+ metadata.gz: b8350e64a62d8fb35540a14a24fce3ae04873c92c13eaae557e8038f57ea6350
4
+ data.tar.gz: 8040f800374683a299abafe6b8c5fa0f4d45d9c9612625a177b2122a3a7343a6
5
5
  SHA512:
6
- metadata.gz: ce2d17a96d496d47848082e7ed980b3c8bd111d071ff256f9788db0d2c13fb41da543eeee39cd3764b6b18d16a53039a49968f5797264be7dcf3b8d472a24aa2
7
- data.tar.gz: 0b86f77306d5b5b79d75d56d013eabf81e0714446a9b8cdf814b66893e4baea1004866060c45d0b2ef1914bf852f7592cde72eba15446d2e881f839e989c7039
6
+ metadata.gz: dc6be2a746153c6e5a2e97a2e2e3b616056cece4ec4dd80fc52f185c44e29214ef7da7a2e4dfb43b3c11edc60d2dc5947a606a56fc0067694ed6196da0bf2391
7
+ data.tar.gz: e3810d24015a37ffd983be9ca9810ce2b3f2aaaa4557592a9a5e3ef179e288e98f7e08abdb17e64391e792bd54bb59e721201dbe6c769428a31961d08294d618
data/.gitignore CHANGED
@@ -26,4 +26,5 @@ coverage/*
26
26
  .vagrant/
27
27
  cov-int/
28
28
  cov-fluentd.tar.gz
29
- .vscode
29
+ .vscode
30
+ .idea/
data/CHANGELOG.md CHANGED
@@ -1,5 +1,42 @@
1
1
  # v1.9
2
2
 
3
+ ## Release v1.9.3 - 2020/03/05
4
+
5
+ ### Enhancement
6
+
7
+ * in_tail: Emit buffered lines as `unmatched_line` at shutdown phase when `emit_unmatched_lines true`
8
+ https://github.com/fluent/fluentd/pull/2837
9
+ * Specify directory mode explicitly
10
+ https://github.com/fluent/fluentd/pull/2827
11
+ * server helper: Change SSLError log level to warn in accept
12
+ https://github.com/fluent/fluentd/pull/2861
13
+ * Refactor code
14
+ https://github.com/fluent/fluentd/pull/2829
15
+ https://github.com/fluent/fluentd/pull/2830
16
+ https://github.com/fluent/fluentd/pull/2832
17
+ https://github.com/fluent/fluentd/pull/2836
18
+ https://github.com/fluent/fluentd/pull/2838
19
+ https://github.com/fluent/fluentd/pull/2842
20
+ https://github.com/fluent/fluentd/pull/2843
21
+
22
+ ### Bug fix
23
+
24
+ * buffer: Add seq to metadata that it can be unique
25
+ https://github.com/fluent/fluentd/pull/2824
26
+ https://github.com/fluent/fluentd/pull/2853
27
+ * buffer: Use `Tempfile` as binmode for decompression
28
+ https://github.com/fluent/fluentd/pull/2847
29
+
30
+ ### Misc
31
+
32
+ * Add `.idea` to git ignore file
33
+ https://github.com/fluent/fluentd/pull/2834
34
+ * appveyor: Fix tests
35
+ https://github.com/fluent/fluentd/pull/2853
36
+ https://github.com/fluent/fluentd/pull/2855
37
+ * Update pem for test
38
+ https://github.com/fluent/fluentd/pull/2839
39
+
3
40
  ## Release v1.9.2 - 2020/02/13
4
41
 
5
42
  ### Enhancement
data/lib/fluent/env.rb CHANGED
@@ -22,6 +22,8 @@ module Fluent
22
22
  DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
23
23
  DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
24
24
  DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
25
+ DEFAULT_DIR_PERMISSION = 0755
26
+ DEFAULT_FILE_PERMISSION = 0644
25
27
 
26
28
  def self.windows?
27
29
  ServerEngine.windows?
@@ -31,8 +31,6 @@ module Fluent
31
31
  DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024 # 256MB
32
32
  DEFAULT_TOTAL_LIMIT_SIZE = 64 * 1024 * 1024 * 1024 # 64GB, same with v0.12 (TimeSlicedOutput + buf_file)
33
33
 
34
- DIR_PERMISSION = 0755
35
-
36
34
  desc 'The path where buffer chunks are stored.'
37
35
  config_param :path, :string, default: nil
38
36
  desc 'The suffix of buffer chunks'
@@ -108,7 +106,7 @@ module Fluent
108
106
  if @dir_permission
109
107
  @dir_permission = @dir_permission.to_i(8) if @dir_permission.is_a?(String)
110
108
  else
111
- @dir_permission = system_config.dir_permission || DIR_PERMISSION
109
+ @dir_permission = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
112
110
  end
113
111
  end
114
112
 
@@ -166,7 +164,17 @@ module Fluent
166
164
 
167
165
  case chunk.state
168
166
  when :staged
169
- stage[chunk.metadata] = chunk
167
+ # unstaged chunk created at Buffer#write_step_by_step is identified as the staged chunk here because FileChunk#assume_chunk_state checks only the file name.
168
+ # https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L663
169
+ # This case can happen when fluentd process is killed by signal or other reasons between creating unstaged chunks and changing them to staged mode in Buffer#write
170
+ # these chunks(unstaged chunks) has shared the same metadata
171
+ # So perform enqueue step again https://github.com/fluent/fluentd/blob/9d113029d4550ce576d8825bfa9612aa3e55bff0/lib/fluent/plugin/buffer.rb#L364
172
+ if chunk_size_full?(chunk) || stage.key?(chunk.metadata)
173
+ chunk.metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
174
+ queue << chunk.enqueued!
175
+ else
176
+ stage[chunk.metadata] = chunk
177
+ end
170
178
  when :queued
171
179
  queue << chunk
172
180
  end
@@ -32,7 +32,6 @@ module Fluent
32
32
  DEFAULT_TOTAL_LIMIT_SIZE = 64 * 1024 * 1024 * 1024 # 64GB
33
33
 
34
34
  PATH_SUFFIX = ".#{Fluent::Plugin::Buffer::FileSingleChunk::PATH_EXT}"
35
- DIR_PERMISSION = 0755
36
35
 
37
36
  desc 'The path where buffer chunks are stored.'
38
37
  config_param :path, :string, default: nil
@@ -128,7 +127,7 @@ module Fluent
128
127
  @dir_permission = if @dir_permission
129
128
  @dir_permission.to_i(8)
130
129
  else
131
- system_config.dir_permission || DIR_PERMISSION
130
+ system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
132
131
  end
133
132
  end
134
133
 
@@ -60,7 +60,17 @@ module Fluent
60
60
  desc 'Compress buffered data.'
61
61
  config_param :compress, :enum, list: [:text, :gzip], default: :text
62
62
 
63
- Metadata = Struct.new(:timekey, :tag, :variables) do
63
+ Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
64
+ def initialize(timekey, tag, variables)
65
+ super(timekey, tag, variables, 0)
66
+ end
67
+
68
+ def dup_next
69
+ m = dup
70
+ m.seq = seq + 1
71
+ m
72
+ end
73
+
64
74
  def empty?
65
75
  timekey.nil? && tag.nil? && variables.nil?
66
76
  end
@@ -353,6 +363,8 @@ module Fluent
353
363
  u = unstaged_chunks[m].pop
354
364
  u.synchronize do
355
365
  if u.unstaged? && !chunk_size_full?(u)
366
+ # `u.metadata.seq` and `m.seq` can be different but Buffer#enqueue_chunk expect them to be the same value
367
+ u.metadata.seq = 0
356
368
  synchronize {
357
369
  @stage[m] = u.staged!
358
370
  @stage_size += u.bytesize
@@ -416,6 +428,7 @@ module Fluent
416
428
  if chunk.empty?
417
429
  chunk.close
418
430
  else
431
+ chunk.metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
419
432
  @queue << chunk
420
433
  @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
421
434
  chunk.enqueued!
@@ -434,6 +447,7 @@ module Fluent
434
447
  synchronize do
435
448
  chunk.synchronize do
436
449
  metadata = chunk.metadata
450
+ metadata.seq = 0 # metadata.seq should be 0 for counting @queued_num
437
451
  @queue << chunk
438
452
  @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
439
453
  chunk.enqueued!
@@ -656,13 +670,15 @@ module Fluent
656
670
  # Then, will generate chunks not staged (not queued) to append rest data.
657
671
  staged_chunk_used = false
658
672
  modified_chunks = []
673
+ modified_metadata = metadata
659
674
  get_next_chunk = ->(){
660
675
  c = if staged_chunk_used
661
676
  # Staging new chunk here is bad idea:
662
677
  # Recovering whole state including newly staged chunks is much harder than current implementation.
663
- generate_chunk(metadata)
678
+ modified_metadata = modified_metadata.dup_next
679
+ generate_chunk(modified_metadata)
664
680
  else
665
- synchronize{ @stage[metadata] ||= generate_chunk(metadata).staged! }
681
+ synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }
666
682
  end
667
683
  modified_chunks << c
668
684
  c
@@ -206,7 +206,7 @@ module Fluent
206
206
  output_io = if chunk_io.is_a?(StringIO)
207
207
  StringIO.new
208
208
  else
209
- Tempfile.new('decompressed-data')
209
+ Tempfile.new('decompressed-data').binmode
210
210
  end
211
211
  decompress(input_io: chunk_io, output_io: output_io)
212
212
  output_io.seek(0, IO::SEEK_SET)
@@ -37,13 +37,11 @@ module Fluent
37
37
  # path_prefix: path prefix string, ended with '.'
38
38
  # path_suffix: path suffix string, like '.log' (or any other user specified)
39
39
 
40
- FILE_PERMISSION = 0644
41
-
42
40
  attr_reader :path, :permission
43
41
 
44
42
  def initialize(metadata, path, mode, perm: nil, compress: :text)
45
43
  super(metadata, compress: compress)
46
- perm ||= FILE_PERMISSION
44
+ perm ||= Fluent::DEFAULT_FILE_PERMISSION
47
45
  @permission = perm.is_a?(String) ? perm.to_i(8) : perm
48
46
  @bytesize = @size = @adding_bytes = @adding_size = 0
49
47
  @meta = nil
@@ -232,6 +230,7 @@ module Fluent
232
230
  @metadata.timekey = data[:timekey]
233
231
  @metadata.tag = data[:tag]
234
232
  @metadata.variables = data[:variables]
233
+ @metadata.seq = data[:seq] || 0
235
234
  end
236
235
 
237
236
  def restore_metadata_partially(chunk)
@@ -243,6 +242,7 @@ module Fluent
243
242
  @metadata.timekey = nil
244
243
  @metadata.tag = nil
245
244
  @metadata.variables = nil
245
+ @metadata.seq = 0
246
246
  end
247
247
 
248
248
  def write_metadata(update: true)
@@ -32,14 +32,13 @@ module Fluent
32
32
  PATH_EXT = 'buf'
33
33
  PATH_SUFFIX = ".#{PATH_EXT}"
34
34
  PATH_REGEXP = /\.(b|q)([0-9a-f]+)\.#{PATH_EXT}*\Z/n # //n switch means explicit 'ASCII-8BIT' pattern
35
- FILE_PERMISSION = 0644
36
35
 
37
36
  attr_reader :path, :permission
38
37
 
39
- def initialize(metadata, path, mode, key, perm: FILE_PERMISSION, compress: :text)
38
+ def initialize(metadata, path, mode, key, perm: Fluent::DEFAULT_FILE_PERMISSION, compress: :text)
40
39
  super(metadata, compress: compress)
41
40
  @key = key
42
- perm ||= FILE_PERMISSION
41
+ perm ||= Fluent::DEFAULT_FILE_PERMISSION
43
42
  @permission = perm.is_a?(String) ? perm.to_i(8) : perm
44
43
  @bytesize = @size = @adding_bytes = @adding_size = 0
45
44
 
@@ -48,8 +48,6 @@ module Fluent::Plugin
48
48
  end
49
49
  end
50
50
 
51
- FILE_PERMISSION = 0644
52
-
53
51
  def initialize
54
52
  super
55
53
  @paths = []
@@ -125,6 +123,8 @@ module Fluent::Plugin
125
123
  parser_config["format#{n}"] = conf["format#{n}"] if conf["format#{n}"]
126
124
  end
127
125
 
126
+ parser_config['unmatched_lines'] = conf['emit_unmatched_lines']
127
+
128
128
  super
129
129
 
130
130
  if !@enable_watch_timer && !@enable_stat_watcher
@@ -167,7 +167,8 @@ module Fluent::Plugin
167
167
  else
168
168
  method(:parse_singleline)
169
169
  end
170
- @file_perm = system_config.file_permission || FILE_PERMISSION
170
+ @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
171
+ @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
171
172
  # parser is already created by parser helper
172
173
  @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
173
174
  end
@@ -210,7 +211,7 @@ module Fluent::Plugin
210
211
 
211
212
  if @pos_file
212
213
  pos_file_dir = File.dirname(@pos_file)
213
- FileUtils.mkdir_p(pos_file_dir) unless Dir.exist?(pos_file_dir)
214
+ FileUtils.mkdir_p(pos_file_dir, mode: @dir_perm) unless Dir.exist?(pos_file_dir)
214
215
  @pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
215
216
  @pf_file.sync = true
216
217
  @pf = PositionFile.load(@pf_file, logger: log)
@@ -317,19 +318,33 @@ module Fluent::Plugin
317
318
  end
318
319
 
319
320
  def setup_watcher(path, pe)
320
- line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
321
- tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @enable_stat_watcher, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
322
- tw.attach do |watcher|
323
- event_loop_attach(watcher.timer_trigger) if watcher.timer_trigger
324
- event_loop_attach(watcher.stat_trigger) if watcher.stat_trigger
321
+ line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
322
+ tw = TailWatcher.new(path, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
323
+
324
+ if @enable_watch_timer
325
+ tt = TimerTrigger.new(1, log) { tw.on_notify }
326
+ tw.register_watcher(tt)
325
327
  end
328
+
329
+ if @enable_stat_watcher
330
+ tt = StatWatcher.new(path, log) { tw.on_notify }
331
+ tw.register_watcher(tt)
332
+ end
333
+
334
+ tw.on_notify
335
+
336
+ tw.watchers.each do |watcher|
337
+ event_loop_attach(watcher)
338
+ end
339
+
326
340
  tw
327
341
  rescue => e
328
342
  if tw
329
- tw.detach { |watcher|
330
- event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
331
- event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
332
- }
343
+ tw.watchers.each do |watcher|
344
+ event_loop_detach(watcher)
345
+ end
346
+
347
+ tw.detach
333
348
  tw.close
334
349
  end
335
350
  raise e
@@ -384,6 +399,8 @@ module Fluent::Plugin
384
399
 
385
400
  # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
386
401
  def update_watcher(path, pe)
402
+ log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
403
+
387
404
  if @pf
388
405
  unless pe.read_inode == @pf[path].read_inode
389
406
  log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
@@ -400,12 +417,13 @@ module Fluent::Plugin
400
417
  # so adding close_io argument to avoid this problem.
401
418
  # At shutdown, IOHandler's io will be released automatically after detached the event loop
402
419
  def detach_watcher(tw, close_io = true)
403
- tw.detach { |watcher|
404
- event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
405
- event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
406
- }
420
+ tw.watchers.each do |watcher|
421
+ event_loop_detach(watcher)
422
+ end
423
+ tw.detach
424
+
407
425
  tw.close if close_io
408
- flush_buffer(tw)
426
+
409
427
  if tw.unwatched && @pf
410
428
  @pf.unwatch(tw.path)
411
429
  end
@@ -419,23 +437,31 @@ module Fluent::Plugin
419
437
  end
420
438
  end
421
439
 
422
- def flush_buffer(tw)
423
- if lb = tw.line_buffer
424
- lb.chomp!
425
- @parser.parse(lb) { |time, record|
426
- if time && record
440
+ def flush_buffer(tw, buf)
441
+ buf.chomp!
442
+ @parser.parse(buf) { |time, record|
443
+ if time && record
444
+ tag = if @tag_prefix || @tag_suffix
445
+ @tag_prefix + tw.tag + @tag_suffix
446
+ else
447
+ @tag
448
+ end
449
+ record[@path_key] ||= tw.path unless @path_key.nil?
450
+ router.emit(tag, time, record)
451
+ else
452
+ if @emit_unmatched_lines
453
+ record = { 'unmatched_line' => buf }
454
+ record[@path_key] ||= tail_watcher.path unless @path_key.nil?
427
455
  tag = if @tag_prefix || @tag_suffix
428
456
  @tag_prefix + tw.tag + @tag_suffix
429
457
  else
430
458
  @tag
431
459
  end
432
- record[@path_key] ||= tw.path unless @path_key.nil?
433
- router.emit(tag, time, record)
434
- else
435
- log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
460
+ router.emit(tag, Fluent::EventTime.now, record)
436
461
  end
437
- }
438
- end
462
+ log.warn "got incomplete line at shutdown from #{tw.path}: #{buf.inspect}"
463
+ end
464
+ }
439
465
  end
440
466
 
441
467
  # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
@@ -490,11 +516,12 @@ module Fluent::Plugin
490
516
  es
491
517
  end
492
518
 
519
+ # No need to check if line_buffer_timer_flusher is nil, since line_buffer_timer_flusher should exist
493
520
  def parse_multilines(lines, tail_watcher)
494
- lb = tail_watcher.line_buffer
521
+ lb = tail_watcher.line_buffer_timer_flusher.line_buffer
495
522
  es = Fluent::MultiEventStream.new
496
523
  if @parser.has_firstline?
497
- tail_watcher.line_buffer_timer_flusher.reset_timer if tail_watcher.line_buffer_timer_flusher
524
+ tail_watcher.line_buffer_timer_flusher.reset_timer
498
525
  lines.each { |line|
499
526
  if @parser.firstline?(line)
500
527
  if lb
@@ -524,26 +551,50 @@ module Fluent::Plugin
524
551
  }
525
552
  end
526
553
  end
527
- tail_watcher.line_buffer = lb
554
+ tail_watcher.line_buffer_timer_flusher.line_buffer = lb
528
555
  es
529
556
  end
530
557
 
558
+ class StatWatcher < Coolio::StatWatcher
559
+ def initialize(path, log, &callback)
560
+ @callback = callback
561
+ @log = log
562
+ super(path)
563
+ end
564
+
565
+ def on_change(prev, cur)
566
+ @callback.call
567
+ rescue
568
+ @log.error $!.to_s
569
+ @log.error_backtrace
570
+ end
571
+ end
572
+
573
+ class TimerTrigger < Coolio::TimerWatcher
574
+ def initialize(interval, log, &callback)
575
+ @log = log
576
+ @callback = callback
577
+ super(interval, true)
578
+ end
579
+
580
+ def on_timer
581
+ @callback.call
582
+ rescue => e
583
+ @log.error e.to_s
584
+ @log.error_backtrace
585
+ end
586
+ end
587
+
531
588
  class TailWatcher
532
- def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, enable_stat_watcher, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
589
+ def initialize(path, pe, log, read_from_head, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
533
590
  @path = path
534
- @rotate_wait = rotate_wait
535
591
  @pe = pe || MemoryPositionEntry.new
536
592
  @read_from_head = read_from_head
537
- @enable_watch_timer = enable_watch_timer
538
- @enable_stat_watcher = enable_stat_watcher
539
593
  @read_lines_limit = read_lines_limit
540
594
  @receive_lines = receive_lines
541
595
  @update_watcher = update_watcher
542
596
 
543
- @stat_trigger = @enable_stat_watcher ? StatWatcher.new(self, &method(:on_notify)) : nil
544
- @timer_trigger = @enable_watch_timer ? TimerTrigger.new(1, log, &method(:on_notify)) : nil
545
-
546
- @rotate_handler = RotateHandler.new(self, &method(:on_rotate))
597
+ @rotate_handler = RotateHandler.new(log, &method(:on_rotate))
547
598
  @io_handler = nil
548
599
  @log = log
549
600
 
@@ -551,32 +602,26 @@ module Fluent::Plugin
551
602
  @from_encoding = from_encoding
552
603
  @encoding = encoding
553
604
  @open_on_every_update = open_on_every_update
605
+ @watchers = []
554
606
  end
555
607
 
556
608
  attr_reader :path
557
- attr_reader :log, :pe, :read_lines_limit, :open_on_every_update
558
- attr_reader :from_encoding, :encoding
559
- attr_reader :stat_trigger, :enable_watch_timer, :enable_stat_watcher
560
- attr_accessor :timer_trigger
561
- attr_accessor :line_buffer, :line_buffer_timer_flusher
609
+ attr_reader :pe
610
+ attr_reader :line_buffer_timer_flusher
562
611
  attr_accessor :unwatched # This is used for removing position entry from PositionFile
612
+ attr_reader :watchers
563
613
 
564
614
  def tag
565
615
  @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
566
616
  end
567
617
 
568
- def wrap_receive_lines(lines)
569
- @receive_lines.call(lines, self)
570
- end
571
-
572
- def attach
573
- on_notify
574
- yield self
618
+ def register_watcher(watcher)
619
+ @watchers << watcher
575
620
  end
576
621
 
577
622
  def detach
578
- yield self
579
623
  @io_handler.on_notify if @io_handler
624
+ @line_buffer_timer_flusher&.close(self)
580
625
  end
581
626
 
582
627
  def close
@@ -629,7 +674,7 @@ module Fluent::Plugin
629
674
  pos = @read_from_head ? 0 : fsize
630
675
  @pe.update(inode, pos)
631
676
  end
632
- @io_handler = IOHandler.new(self, &method(:wrap_receive_lines))
677
+ @io_handler = io_handler
633
678
  else
634
679
  @io_handler = NullIOHandler.new
635
680
  end
@@ -654,18 +699,21 @@ module Fluent::Plugin
654
699
  watcher_needs_update = true
655
700
  end
656
701
 
657
- log_msg = "detected rotation of #{@path}"
658
- log_msg << "; waiting #{@rotate_wait} seconds" if watcher_needs_update # wait rotate_time if previous file exists
659
- @log.info log_msg
660
-
661
702
  if watcher_needs_update
662
703
  @update_watcher.call(@path, swap_state(@pe))
663
704
  else
664
- @io_handler = IOHandler.new(self, &method(:wrap_receive_lines))
705
+ @log.info "detected rotation of #{@path}"
706
+ @io_handler = io_handler
665
707
  end
666
708
  end
667
709
  end
668
710
 
711
+ def io_handler
712
+ IOHandler.new(self, path: @path, log: @log, read_lines_limit: @read_lines_limit, open_on_every_update: @open_on_every_update, from_encoding: @from_encoding, encoding: @encoding) do |lines|
713
+ @receive_lines.call(lines, self)
714
+ end
715
+ end
716
+
669
717
  def swap_state(pe)
670
718
  # Use MemoryPositionEntry for rotated file temporary
671
719
  mpe = MemoryPositionEntry.new
@@ -674,37 +722,6 @@ module Fluent::Plugin
674
722
  pe # This pe will be updated in on_rotate after TailWatcher is initialized
675
723
  end
676
724
 
677
- class TimerTrigger < Coolio::TimerWatcher
678
- def initialize(interval, log, &callback)
679
- @callback = callback
680
- @log = log
681
- super(interval, true)
682
- end
683
-
684
- def on_timer
685
- @callback.call
686
- rescue => e
687
- @log.error e.to_s
688
- @log.error_backtrace
689
- end
690
- end
691
-
692
- class StatWatcher < Coolio::StatWatcher
693
- def initialize(watcher, &callback)
694
- @watcher = watcher
695
- @callback = callback
696
- super(watcher.path)
697
- end
698
-
699
- def on_change(prev, cur)
700
- @callback.call
701
- rescue
702
- # TODO log?
703
- @watcher.log.error $!.to_s
704
- @watcher.log.error_backtrace
705
- end
706
- end
707
-
708
725
  class FIFO
709
726
  def initialize(from_encoding, encoding)
710
727
  @from_encoding = from_encoding
@@ -764,15 +781,20 @@ module Fluent::Plugin
764
781
  end
765
782
 
766
783
  class IOHandler
767
- def initialize(watcher, &receive_lines)
784
+ def initialize(watcher, path:, read_lines_limit:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
768
785
  @watcher = watcher
786
+ @path = path
787
+ @read_lines_limit = read_lines_limit
769
788
  @receive_lines = receive_lines
770
- @fifo = FIFO.new(@watcher.from_encoding || Encoding::ASCII_8BIT, @watcher.encoding || Encoding::ASCII_8BIT)
789
+ @open_on_every_update = open_on_every_update
790
+ @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
771
791
  @iobuf = ''.force_encoding('ASCII-8BIT')
772
792
  @lines = []
773
793
  @io = nil
774
794
  @notify_mutex = Mutex.new
775
- @watcher.log.info "following tail of #{@watcher.path}"
795
+ @log = log
796
+
797
+ @log.info "following tail of #{@path}"
776
798
  end
777
799
 
778
800
  def on_notify
@@ -789,7 +811,7 @@ module Fluent::Plugin
789
811
  while true
790
812
  @fifo << io.readpartial(8192, @iobuf)
791
813
  @fifo.read_lines(@lines)
792
- if @lines.size >= @watcher.read_lines_limit
814
+ if @lines.size >= @read_lines_limit
793
815
  # not to use too much memory in case the file is very large
794
816
  read_more = true
795
817
  break
@@ -823,37 +845,35 @@ module Fluent::Plugin
823
845
  end
824
846
 
825
847
  def open
826
- io = Fluent::FileWrapper.open(@watcher.path)
848
+ io = Fluent::FileWrapper.open(@path)
827
849
  io.seek(@watcher.pe.read_pos + @fifo.bytesize)
828
850
  io
829
851
  rescue RangeError
830
852
  io.close if io
831
- raise WatcherSetupError, "seek error with #{@watcher.path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
853
+ raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
832
854
  rescue Errno::ENOENT
833
855
  nil
834
856
  end
835
857
 
836
858
  def with_io
837
- begin
838
- if @watcher.open_on_every_update
839
- io = open
840
- begin
841
- yield io
842
- ensure
843
- io.close unless io.nil?
844
- end
845
- else
846
- @io ||= open
847
- yield @io
859
+ if @open_on_every_update
860
+ io = open
861
+ begin
862
+ yield io
863
+ ensure
864
+ io.close unless io.nil?
848
865
  end
849
- rescue WatcherSetupError => e
850
- close
851
- raise e
852
- rescue
853
- @watcher.log.error $!.to_s
854
- @watcher.log.error_backtrace
855
- close
866
+ else
867
+ @io ||= open
868
+ yield @io
856
869
  end
870
+ rescue WatcherSetupError => e
871
+ close
872
+ raise e
873
+ rescue
874
+ @log.error $!.to_s
875
+ @log.error_backtrace
876
+ close
857
877
  end
858
878
  end
859
879
 
@@ -876,8 +896,8 @@ module Fluent::Plugin
876
896
  end
877
897
 
878
898
  class RotateHandler
879
- def initialize(watcher, &on_rotate)
880
- @watcher = watcher
899
+ def initialize(log, &on_rotate)
900
+ @log = log
881
901
  @inode = nil
882
902
  @fsize = -1 # first
883
903
  @on_rotate = on_rotate
@@ -892,39 +912,50 @@ module Fluent::Plugin
892
912
  fsize = stat.size
893
913
  end
894
914
 
895
- begin
896
- if @inode != inode || fsize < @fsize
897
- @on_rotate.call(stat)
898
- end
899
- @inode = inode
900
- @fsize = fsize
915
+ if @inode != inode || fsize < @fsize
916
+ @on_rotate.call(stat)
901
917
  end
902
-
918
+ @inode = inode
919
+ @fsize = fsize
903
920
  rescue
904
- @watcher.log.error $!.to_s
905
- @watcher.log.error_backtrace
921
+ @log.error $!.to_s
922
+ @log.error_backtrace
906
923
  end
907
924
  end
908
925
 
909
926
  class LineBufferTimerFlusher
927
+ attr_accessor :line_buffer
928
+
910
929
  def initialize(log, flush_interval, &flush_method)
911
930
  @log = log
912
931
  @flush_interval = flush_interval
913
932
  @flush_method = flush_method
914
933
  @start = nil
934
+ @line_buffer = nil
915
935
  end
916
936
 
917
937
  def on_notify(tw)
918
- if @start && @flush_interval
919
- if Time.now - @start >= @flush_interval
920
- @flush_method.call(tw)
921
- tw.line_buffer = nil
922
- @start = nil
923
- end
938
+ unless @start && @flush_method
939
+ return
940
+ end
941
+
942
+ if Time.now - @start >= @flush_interval
943
+ @flush_method.call(tw, @line_buffer) if @line_buffer
944
+ @line_buffer = nil
945
+ @start = nil
924
946
  end
925
947
  end
926
948
 
949
+ def close(tw)
950
+ return unless @line_buffer
951
+
952
+ @flush_method.call(tw, @line_buffer)
953
+ @line_buffer = nil
954
+ end
955
+
927
956
  def reset_timer
957
+ return unless @flush_interval
958
+
928
959
  @start = Time.now
929
960
  end
930
961
  end