fluentd 0.14.11-x86-mingw32 → 0.14.12-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -5
  3. data/ChangeLog +54 -2
  4. data/example/in_dummy_blocks.conf +17 -0
  5. data/example/in_forward_tls.conf +14 -0
  6. data/example/in_forward_workers.conf +21 -0
  7. data/example/logevents.conf +25 -0
  8. data/example/out_forward_heartbeat_none.conf +16 -0
  9. data/example/out_forward_tls.conf +18 -0
  10. data/example/suppress_config_dump.conf +7 -0
  11. data/lib/fluent/agent.rb +3 -32
  12. data/lib/fluent/clock.rb +62 -0
  13. data/lib/fluent/command/fluentd.rb +12 -0
  14. data/lib/fluent/compat/input.rb +10 -1
  15. data/lib/fluent/compat/output.rb +40 -1
  16. data/lib/fluent/config/configure_proxy.rb +30 -7
  17. data/lib/fluent/config/section.rb +4 -0
  18. data/lib/fluent/config/types.rb +2 -2
  19. data/lib/fluent/configurable.rb +31 -5
  20. data/lib/fluent/engine.rb +61 -12
  21. data/lib/fluent/event_router.rb +6 -0
  22. data/lib/fluent/load.rb +0 -1
  23. data/lib/fluent/log.rb +118 -42
  24. data/lib/fluent/match.rb +37 -0
  25. data/lib/fluent/plugin.rb +25 -3
  26. data/lib/fluent/plugin/base.rb +4 -0
  27. data/lib/fluent/plugin/buf_file.rb +38 -14
  28. data/lib/fluent/plugin/buffer.rb +20 -20
  29. data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
  30. data/lib/fluent/plugin/compressable.rb +1 -0
  31. data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
  32. data/lib/fluent/plugin/formatter_csv.rb +4 -1
  33. data/lib/fluent/plugin/formatter_hash.rb +5 -1
  34. data/lib/fluent/plugin/formatter_json.rb +10 -0
  35. data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
  36. data/lib/fluent/plugin/in_dummy.rb +4 -0
  37. data/lib/fluent/plugin/in_exec.rb +4 -0
  38. data/lib/fluent/plugin/in_forward.rb +11 -3
  39. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  40. data/lib/fluent/plugin/in_http.rb +4 -0
  41. data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
  42. data/lib/fluent/plugin/in_object_space.rb +4 -1
  43. data/lib/fluent/plugin/in_syslog.rb +4 -0
  44. data/lib/fluent/plugin/in_tail.rb +193 -116
  45. data/lib/fluent/plugin/in_tcp.rb +5 -1
  46. data/lib/fluent/plugin/in_udp.rb +4 -0
  47. data/lib/fluent/plugin/input.rb +4 -0
  48. data/lib/fluent/plugin/out_copy.rb +4 -0
  49. data/lib/fluent/plugin/out_exec.rb +4 -0
  50. data/lib/fluent/plugin/out_exec_filter.rb +4 -0
  51. data/lib/fluent/plugin/out_file.rb +70 -30
  52. data/lib/fluent/plugin/out_forward.rb +132 -28
  53. data/lib/fluent/plugin/out_null.rb +10 -0
  54. data/lib/fluent/plugin/out_relabel.rb +4 -0
  55. data/lib/fluent/plugin/out_roundrobin.rb +4 -0
  56. data/lib/fluent/plugin/out_secondary_file.rb +5 -0
  57. data/lib/fluent/plugin/out_stdout.rb +5 -0
  58. data/lib/fluent/plugin/output.rb +18 -9
  59. data/lib/fluent/plugin/storage_local.rb +25 -2
  60. data/lib/fluent/plugin_helper/cert_option.rb +159 -0
  61. data/lib/fluent/plugin_helper/child_process.rb +6 -6
  62. data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
  63. data/lib/fluent/plugin_helper/event_loop.rb +29 -4
  64. data/lib/fluent/plugin_helper/inject.rb +14 -1
  65. data/lib/fluent/plugin_helper/server.rb +275 -31
  66. data/lib/fluent/plugin_helper/socket.rb +144 -4
  67. data/lib/fluent/plugin_helper/socket_option.rb +2 -17
  68. data/lib/fluent/plugin_helper/storage.rb +7 -1
  69. data/lib/fluent/plugin_helper/thread.rb +16 -4
  70. data/lib/fluent/registry.rb +26 -9
  71. data/lib/fluent/root_agent.rb +7 -3
  72. data/lib/fluent/supervisor.rb +37 -15
  73. data/lib/fluent/system_config.rb +37 -10
  74. data/lib/fluent/test.rb +2 -0
  75. data/lib/fluent/test/driver/base.rb +24 -26
  76. data/lib/fluent/test/helpers.rb +21 -0
  77. data/lib/fluent/version.rb +1 -1
  78. data/test/command/test_fluentd.rb +274 -4
  79. data/test/config/test_configurable.rb +154 -0
  80. data/test/config/test_configure_proxy.rb +180 -1
  81. data/test/config/test_system_config.rb +10 -0
  82. data/test/config/test_types.rb +1 -0
  83. data/test/plugin/test_base.rb +4 -0
  84. data/test/plugin/test_buf_file.rb +241 -9
  85. data/test/plugin/test_buffer.rb +11 -11
  86. data/test/plugin/test_buffer_file_chunk.rb +6 -6
  87. data/test/plugin/test_compressable.rb +3 -0
  88. data/test/plugin/test_filter.rb +4 -0
  89. data/test/plugin/test_filter_record_transformer.rb +20 -0
  90. data/test/plugin/test_formatter_csv.rb +9 -0
  91. data/test/plugin/test_formatter_hash.rb +35 -0
  92. data/test/plugin/test_formatter_json.rb +8 -0
  93. data/test/plugin/test_formatter_ltsv.rb +7 -0
  94. data/test/plugin/test_in_dummy.rb +7 -3
  95. data/test/plugin/test_in_monitor_agent.rb +43 -5
  96. data/test/plugin/test_in_tail.rb +97 -4
  97. data/test/plugin/test_input.rb +4 -0
  98. data/test/plugin/test_out_file.rb +46 -7
  99. data/test/plugin/test_out_forward.rb +59 -7
  100. data/test/plugin/test_output.rb +10 -4
  101. data/test/plugin/test_output_as_buffered.rb +37 -25
  102. data/test/plugin/test_output_as_buffered_compress.rb +1 -1
  103. data/test/plugin/test_output_as_buffered_retries.rb +6 -6
  104. data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
  105. data/test/plugin/test_storage_local.rb +40 -1
  106. data/test/plugin_helper/test_child_process.rb +29 -28
  107. data/test/plugin_helper/test_compat_parameters.rb +1 -1
  108. data/test/plugin_helper/test_inject.rb +27 -9
  109. data/test/plugin_helper/test_server.rb +822 -50
  110. data/test/plugin_helper/test_storage.rb +11 -0
  111. data/test/plugin_helper/test_timer.rb +1 -0
  112. data/test/test_clock.rb +164 -0
  113. data/test/test_log.rb +146 -15
  114. data/test/test_plugin.rb +251 -0
  115. data/test/test_supervisor.rb +65 -57
  116. data/test/test_test_drivers.rb +2 -2
  117. metadata +18 -7
  118. data/lib/fluent/process.rb +0 -504
  119. data/test/test_process.rb +0 -48
@@ -25,6 +25,7 @@ module Fluent
25
25
 
26
26
  config_param :delimiter, :string, default: "\t"
27
27
  config_param :label_delimiter, :string, default: ":"
28
+ config_param :add_newline, :bool, default: true
28
29
 
29
30
  # TODO: escaping for \t in values
30
31
  def format(tag, time, record)
@@ -33,7 +34,7 @@ module Fluent
33
34
  formatted << @delimiter if formatted.length.nonzero?
34
35
  formatted << "#{label}#{@label_delimiter}#{value}"
35
36
  end
36
- formatted << "\n"
37
+ formatted << "\n".freeze if @add_newline
37
38
  formatted
38
39
  end
39
40
  end
@@ -67,6 +67,10 @@ module Fluent::Plugin
67
67
  @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
68
68
  end
69
69
 
70
+ def multi_workers_ready?
71
+ true
72
+ end
73
+
70
74
  def start
71
75
  super
72
76
 
@@ -64,6 +64,10 @@ module Fluent::Plugin
64
64
  @parser = parser_create
65
65
  end
66
66
 
67
+ def multi_workers_ready?
68
+ true
69
+ end
70
+
67
71
  def start
68
72
  super
69
73
 
@@ -19,6 +19,7 @@ require 'fluent/plugin/input'
19
19
  require 'fluent/msgpack_factory'
20
20
  require 'yajl'
21
21
  require 'digest'
22
+ require 'securerandom'
22
23
 
23
24
  module Fluent::Plugin
24
25
  class ForwardInput < Input
@@ -142,22 +143,29 @@ module Fluent::Plugin
142
143
  end
143
144
  end
144
145
 
146
+ def multi_workers_ready?
147
+ true
148
+ end
149
+
145
150
  HEARTBEAT_UDP_PAYLOAD = "\0"
146
151
 
147
152
  def start
148
153
  super
149
154
 
155
+ shared_socket = system_config.workers > 1
156
+
157
+ log.info "listening port", port: @port, bind: @bind
150
158
  server_create_connection(
151
159
  :in_forward_server, @port,
152
160
  bind: @bind,
153
- shared: false,
161
+ shared: shared_socket,
154
162
  resolve_name: @resolve_hostname,
155
163
  linger_timeout: @linger_timeout,
156
164
  backlog: @backlog,
157
165
  &method(:handle_connection)
158
166
  )
159
167
 
160
- server_create(:in_forward_server_udp_heartbeat, @port, shared: false, proto: :udp, bind: @bind, resolve_name: @resolve_hostname, max_bytes: 128) do |data, sock|
168
+ server_create(:in_forward_server_udp_heartbeat, @port, shared: shared_socket, proto: :udp, bind: @bind, resolve_name: @resolve_hostname, max_bytes: 128) do |data, sock|
161
169
  log.trace "heartbeat udp data arrived", host: sock.remote_host, port: sock.remote_port, data: data
162
170
  begin
163
171
  sock.write HEARTBEAT_UDP_PAYLOAD
@@ -387,7 +395,7 @@ module Fluent::Plugin
387
395
  end
388
396
 
389
397
  def generate_salt
390
- OpenSSL::Random.random_bytes(16)
398
+ ::SecureRandom.random_bytes(16)
391
399
  end
392
400
 
393
401
  def generate_helo(nonce, user_auth_salt)
@@ -33,6 +33,10 @@ module Fluent::Plugin
33
33
  super
34
34
  end
35
35
 
36
+ def multi_workers_ready?
37
+ true
38
+ end
39
+
36
40
  def start
37
41
  super
38
42
 
@@ -113,6 +113,10 @@ module Fluent::Plugin
113
113
  end
114
114
  end
115
115
 
116
+ def multi_workers_ready?
117
+ true
118
+ end
119
+
116
120
  def start
117
121
  @_event_loop_run_timeout = @blocking_timeout
118
122
 
@@ -85,6 +85,10 @@ module Fluent::Plugin
85
85
  opts[:pretty_json] = true
86
86
  end
87
87
 
88
+ if ivars = (qs['with_ivars'] || []).first
89
+ opts[:ivars] = ivars.split(',')
90
+ end
91
+
88
92
  if with_config = get_search_parameter(qs, 'with_config'.freeze)
89
93
  opts[:with_config] = Fluent::Config.bool_value(with_config)
90
94
  end
@@ -216,6 +220,16 @@ module Fluent::Plugin
216
220
  end
217
221
  end
218
222
 
223
+ def initialize
224
+ super
225
+
226
+ @first_warn = false
227
+ end
228
+
229
+ def multi_workers_ready?
230
+ true
231
+ end
232
+
219
233
  def start
220
234
  super
221
235
 
@@ -259,8 +273,8 @@ module Fluent::Plugin
259
273
 
260
274
  MONITOR_INFO = {
261
275
  'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
262
- 'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer); @buffer.queue.size },
263
- 'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer); @buffer.stage_size },
276
+ 'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil?; @buffer.queue.size },
277
+ 'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil?; @buffer.stage_size },
264
278
  'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
265
279
  }
266
280
 
@@ -346,6 +360,12 @@ module Fluent::Plugin
346
360
  catch(:skip) do
347
361
  obj[key] = pe.instance_exec(&code)
348
362
  end
363
+ rescue NoMethodError => e
364
+ unless @first_warn
365
+ log.error "NoMethodError in monitoring plugins", key: key, plugin: pe.class, error: e
366
+ log.error_backtrace
367
+ @first_warn = true
368
+ end
349
369
  rescue => e
350
370
  log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
351
371
  end
@@ -364,6 +384,13 @@ module Fluent::Plugin
364
384
  }
365
385
  end
366
386
  obj['instance_variables'] = iv
387
+ elsif ivars = opts[:ivars]
388
+ iv = {}
389
+ ivars.each {|name|
390
+ iname = "@#{name}"
391
+ iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)
392
+ }
393
+ obj['instance_variables'] = iv
367
394
  end
368
395
 
369
396
  obj
@@ -14,7 +14,6 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'cool.io'
18
17
  require 'yajl'
19
18
 
20
19
  require 'fluent/plugin/input'
@@ -33,6 +32,10 @@ module Fluent::Plugin
33
32
  config_param :tag, :string
34
33
  config_param :top, :integer, default: 15
35
34
 
35
+ def multi_workers_ready?
36
+ true
37
+ end
38
+
36
39
  def start
37
40
  super
38
41
 
@@ -120,6 +120,10 @@ module Fluent::Plugin
120
120
  @_event_loop_run_timeout = @blocking_timeout
121
121
  end
122
122
 
123
+ def multi_workers_ready?
124
+ true
125
+ end
126
+
123
127
  def start
124
128
  super
125
129
 
@@ -65,6 +65,8 @@ module Fluent::Plugin
65
65
  config_param :read_lines_limit, :integer, default: 1000
66
66
  desc 'The interval of flushing the buffer for multiline format'
67
67
  config_param :multiline_flush_interval, :time, default: nil
68
+ desc 'Enable the option to emit unmatched lines.'
69
+ config_param :emit_unmatched_lines, :bool, default: false
68
70
  desc 'Enable the additional watch timer.'
69
71
  config_param :enable_watch_timer, :bool, default: true
70
72
  desc 'The encoding after conversion of the input.'
@@ -73,6 +75,8 @@ module Fluent::Plugin
73
75
  config_param :from_encoding, :string, default: nil
74
76
  desc 'Add the log path being tailed to records. Specify the field name to be used.'
75
77
  config_param :path_key, :string, default: nil
78
+ desc 'Open and close the file on every update instead of leaving it open until it gets rotated.'
79
+ config_param :open_on_every_update, :bool, default: false
76
80
 
77
81
  attr_reader :paths
78
82
 
@@ -213,7 +217,7 @@ module Fluent::Plugin
213
217
 
214
218
  def setup_watcher(path, pe)
215
219
  line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
216
- tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, &method(:receive_lines))
220
+ tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
217
221
  tw.attach do |watcher|
218
222
  watcher.timer_trigger = timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
219
223
  event_loop_attach(watcher.stat_trigger)
@@ -297,13 +301,6 @@ module Fluent::Plugin
297
301
  def flush_buffer(tw)
298
302
  if lb = tw.line_buffer
299
303
  lb.chomp!
300
- if @encoding
301
- if @from_encoding
302
- lb.encode!(@encoding, @from_encoding)
303
- else
304
- lb.force_encoding(@encoding)
305
- end
306
- end
307
304
  @parser.parse(lb) { |time, record|
308
305
  if time && record
309
306
  tag = if @tag_prefix || @tag_suffix
@@ -345,18 +342,16 @@ module Fluent::Plugin
345
342
  def convert_line_to_event(line, es, tail_watcher)
346
343
  begin
347
344
  line.chomp! # remove \n
348
- if @encoding
349
- if @from_encoding
350
- line.encode!(@encoding, @from_encoding)
351
- else
352
- line.force_encoding(@encoding)
353
- end
354
- end
355
345
  @parser.parse(line) { |time, record|
356
346
  if time && record
357
347
  record[@path_key] ||= tail_watcher.path unless @path_key.nil?
358
348
  es.add(time, record)
359
349
  else
350
+ if @emit_unmatched_lines
351
+ record = {'unmatched_line' => line}
352
+ record[@path_key] ||= tail_watcher.path unless @path_key.nil?
353
+ es.add(Fluent::EventTime.now, record)
354
+ end
360
355
  log.warn "pattern not match: #{line.inspect}"
361
356
  end
362
357
  }
@@ -387,6 +382,9 @@ module Fluent::Plugin
387
382
  lb = line
388
383
  else
389
384
  if lb.nil?
385
+ if @emit_unmatched_lines
386
+ convert_line_to_event(line, es, tail_watcher)
387
+ end
390
388
  log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
391
389
  else
392
390
  lb << line
@@ -410,7 +408,7 @@ module Fluent::Plugin
410
408
  end
411
409
 
412
410
  class TailWatcher
413
- def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
411
+ def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
414
412
  @path = path
415
413
  @rotate_wait = rotate_wait
416
414
  @pe = pe || MemoryPositionEntry.new
@@ -420,17 +418,22 @@ module Fluent::Plugin
420
418
  @receive_lines = receive_lines
421
419
  @update_watcher = update_watcher
422
420
 
423
- @stat_trigger = StatWatcher.new(path, log, &method(:on_notify))
421
+ @stat_trigger = StatWatcher.new(self, &method(:on_notify))
424
422
  @timer_trigger = nil
425
423
 
426
- @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
424
+ @rotate_handler = RotateHandler.new(self, &method(:on_rotate))
427
425
  @io_handler = nil
428
426
  @log = log
429
427
 
430
428
  @line_buffer_timer_flusher = line_buffer_timer_flusher
429
+ @from_encoding = from_encoding
430
+ @encoding = encoding
431
+ @open_on_every_update = open_on_every_update
431
432
  end
432
433
 
433
434
  attr_reader :path
435
+ attr_reader :log, :pe, :read_lines_limit, :open_on_every_update
436
+ attr_reader :from_encoding, :encoding
434
437
  attr_reader :stat_trigger, :enable_watch_timer
435
438
  attr_accessor :timer_trigger
436
439
  attr_accessor :line_buffer, :line_buffer_timer_flusher
@@ -458,21 +461,27 @@ module Fluent::Plugin
458
461
  def close
459
462
  if @io_handler
460
463
  @io_handler.close
464
+ @io_handler = nil
461
465
  end
462
466
  end
463
467
 
464
468
  def on_notify
465
- @rotate_handler.on_notify if @rotate_handler
469
+ begin
470
+ stat = Fluent::FileWrapper.stat(@path)
471
+ rescue Errno::ENOENT
472
+ # moved or deleted
473
+ stat = nil
474
+ end
475
+
476
+ @rotate_handler.on_notify(stat) if @rotate_handler
466
477
  @line_buffer_timer_flusher.on_notify(self) if @line_buffer_timer_flusher
467
- return unless @io_handler
468
- @io_handler.on_notify
478
+ @io_handler.on_notify if @io_handler
469
479
  end
470
480
 
471
- def on_rotate(io)
472
- if @io_handler == nil
473
- if io
481
+ def on_rotate(stat)
482
+ if @io_handler.nil?
483
+ if stat
474
484
  # first time
475
- stat = io.stat
476
485
  fsize = stat.size
477
486
  inode = stat.ino
478
487
 
@@ -483,13 +492,11 @@ module Fluent::Plugin
483
492
  # a) file was once renamed and backed, or
484
493
  # b) symlink or hardlink to the same file is recreated
485
494
  # in either case, seek to the saved position
486
- pos = @pe.read_pos
487
495
  elsif last_inode != 0
488
496
  # this is FilePositionEntry and fluentd once started.
489
497
  # read data from the head of the rotated file.
490
498
  # logs never duplicate because this file is a rotated new file.
491
- pos = 0
492
- @pe.update(inode, pos)
499
+ @pe.update(inode, 0)
493
500
  else
494
501
  # this is MemoryPositionEntry or this is the first time fluentd started.
495
502
  # seek to the end of the any files.
@@ -498,36 +505,37 @@ module Fluent::Plugin
498
505
  pos = @read_from_head ? 0 : fsize
499
506
  @pe.update(inode, pos)
500
507
  end
501
- io.seek(pos)
502
-
503
- @io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
508
+ @io_handler = IOHandler.new(self, &method(:wrap_receive_lines))
504
509
  else
505
510
  @io_handler = NullIOHandler.new
506
511
  end
507
512
  else
508
- log_msg = "detected rotation of #{@path}"
509
- log_msg << "; waiting #{@rotate_wait} seconds" if @io_handler.io # wait rotate_time if previous file is exist
510
- @log.info log_msg
513
+ watcher_needs_update = false
511
514
 
512
- if io
513
- stat = io.stat
515
+ if stat
514
516
  inode = stat.ino
515
517
  if inode == @pe.read_inode # truncated
516
- @pe.update_pos(stat.size)
517
- io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
518
+ @pe.update_pos(0)
518
519
  @io_handler.close
519
- @io_handler = io_handler
520
- elsif @io_handler.io.nil? # There is no previous file. Reuse TailWatcher
521
- @pe.update(inode, io.pos)
522
- io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
523
- @io_handler = io_handler
520
+ elsif !@io_handler.opened? # There is no previous file. Reuse TailWatcher
521
+ @pe.update(inode, 0)
524
522
  else # file is rotated and new file found
525
- @update_watcher.call(@path, swap_state(@pe))
523
+ watcher_needs_update = true
526
524
  end
527
525
  else # file is rotated and new file not found
528
526
  # Clear RotateHandler to avoid duplicated file watch in same path.
529
527
  @rotate_handler = nil
528
+ watcher_needs_update = true
529
+ end
530
+
531
+ log_msg = "detected rotation of #{@path}"
532
+ log_msg << "; waiting #{@rotate_wait} seconds" if watcher_needs_update # wait rotate_time if previous file exists
533
+ @log.info log_msg
534
+
535
+ if watcher_needs_update
530
536
  @update_watcher.call(@path, swap_state(@pe))
537
+ else
538
+ @io_handler = IOHandler.new(self, &method(:wrap_receive_lines))
531
539
  end
532
540
  end
533
541
  end
@@ -537,86 +545,154 @@ module Fluent::Plugin
537
545
  mpe = MemoryPositionEntry.new
538
546
  mpe.update(pe.read_inode, pe.read_pos)
539
547
  @pe = mpe
540
- @io_handler.pe = mpe # Don't re-create IOHandler because IOHandler has an internal buffer.
541
-
542
548
  pe # This pe will be updated in on_rotate after TailWatcher is initialized
543
549
  end
544
550
 
545
551
  class StatWatcher < Coolio::StatWatcher
546
- def initialize(path, log, &callback)
552
+ def initialize(watcher, &callback)
553
+ @watcher = watcher
547
554
  @callback = callback
548
- @log = log
549
- super(path)
555
+ super(watcher.path)
550
556
  end
551
557
 
552
558
  def on_change(prev, cur)
553
559
  @callback.call
554
560
  rescue
555
561
  # TODO log?
556
- @log.error $!.to_s
557
- @log.error_backtrace
562
+ @watcher.log.error $!.to_s
563
+ @watcher.log.error_backtrace
564
+ end
565
+ end
566
+
567
+
568
+ class FIFO
569
+ def initialize(from_encoding, encoding)
570
+ @from_encoding = from_encoding
571
+ @encoding = encoding
572
+ @buffer = ''.force_encoding(from_encoding)
573
+ @eol = "\n".encode(from_encoding).freeze
574
+ end
575
+
576
+ attr_reader :from_encoding, :encoding, :buffer
577
+
578
+ def <<(chunk)
579
+ # Although "chunk" is most likely transient besides String#force_encoding itself
580
+ # won't affect the actual content of it, it is also probable that "chunk" is
581
+ # a reused buffer and changing its encoding causes some problems on the caller side.
582
+ #
583
+ # Actually, the caller here is specific and "chunk" comes from IO#partial with
584
+ # the second argument, which the function always returns as a return value.
585
+ #
586
+ # Feeding a string that has its encoding attribute set to any double-byte or
587
+ # quad-byte encoding to IO#readpartial as the second arguments results in an
588
+ # assertion failure on Ruby < 2.4.0 for unknown reasons.
589
+ orig_encoding = chunk.encoding
590
+ chunk.force_encoding(from_encoding)
591
+ @buffer << chunk
592
+ # Thus the encoding needs to be reverted back here
593
+ chunk.force_encoding(orig_encoding)
594
+ end
595
+
596
+ def convert(s)
597
+ if @from_encoding == @encoding
598
+ s
599
+ else
600
+ s.encode(@encoding, @from_encoding)
601
+ end
602
+ end
603
+
604
+ def next_line
605
+ idx = @buffer.index(@eol)
606
+ convert(@buffer.slice!(0, idx + 1)) unless idx.nil?
607
+ end
608
+
609
+ def bytesize
610
+ @buffer.bytesize
558
611
  end
559
612
  end
560
613
 
561
614
  class IOHandler
562
- def initialize(io, pe, log, read_lines_limit, first = true, &receive_lines)
563
- @log = log
564
- @log.info "following tail of #{io.path}" if first
565
- @io = io
566
- @pe = pe
567
- @read_lines_limit = read_lines_limit
615
+ def initialize(watcher, &receive_lines)
616
+ @watcher = watcher
568
617
  @receive_lines = receive_lines
569
- @buffer = ''.force_encoding('ASCII-8BIT')
618
+ @fifo = FIFO.new(@watcher.from_encoding || Encoding::ASCII_8BIT, @watcher.encoding || Encoding::ASCII_8BIT)
570
619
  @iobuf = ''.force_encoding('ASCII-8BIT')
571
620
  @lines = []
621
+ @io = nil
622
+ @watcher.log.info "following tail of #{@watcher.path}"
572
623
  end
573
624
 
574
- attr_reader :io
575
- attr_accessor :pe
576
-
577
625
  def on_notify
578
- begin
579
- read_more = false
580
-
581
- if @lines.empty?
582
- begin
583
- while true
584
- if @buffer.empty?
585
- @io.readpartial(2048, @buffer)
586
- else
587
- @buffer << @io.readpartial(2048, @iobuf)
588
- end
589
- while idx = @buffer.index("\n".freeze)
590
- @lines << @buffer.slice!(0, idx + 1)
591
- end
592
- if @lines.size >= @read_lines_limit
593
- # not to use too much memory in case the file is very large
594
- read_more = true
595
- break
626
+ with_io do |io|
627
+ begin
628
+ read_more = false
629
+
630
+ if !io.nil? && @lines.empty?
631
+ begin
632
+ while true
633
+ @fifo << io.readpartial(2048, @iobuf)
634
+ while (line = @fifo.next_line)
635
+ @lines << line
636
+ end
637
+ if @lines.size >= @watcher.read_lines_limit
638
+ # not to use too much memory in case the file is very large
639
+ read_more = true
640
+ break
641
+ end
596
642
  end
643
+ rescue EOFError
597
644
  end
598
- rescue EOFError
599
645
  end
600
- end
601
646
 
602
- unless @lines.empty?
603
- if @receive_lines.call(@lines)
604
- @pe.update_pos(@io.pos - @buffer.bytesize)
605
- @lines.clear
606
- else
607
- read_more = false
647
+ unless @lines.empty?
648
+ if @receive_lines.call(@lines)
649
+ @watcher.pe.update_pos(io.pos - @fifo.bytesize)
650
+ @lines.clear
651
+ else
652
+ read_more = false
653
+ end
608
654
  end
609
- end
610
- end while read_more
611
-
612
- rescue
613
- @log.error $!.to_s
614
- @log.error_backtrace
615
- close
655
+ end while read_more
656
+ end
616
657
  end
617
658
 
618
659
  def close
619
- @io.close unless @io.closed?
660
+ if @io && !@io.closed?
661
+ @io.close
662
+ @io = nil
663
+ end
664
+ end
665
+
666
+ def opened?
667
+ !!@io
668
+ end
669
+
670
+ def open
671
+ io = Fluent::FileWrapper.open(@watcher.path)
672
+ io.seek(@watcher.pe.read_pos + @fifo.bytesize)
673
+ io
674
+ rescue Errno::ENOENT
675
+ nil
676
+ end
677
+
678
+ def with_io
679
+ begin
680
+ if @watcher.open_on_every_update
681
+ io = open
682
+ begin
683
+ yield io
684
+ ensure
685
+ io.close unless io.nil?
686
+ end
687
+ else
688
+ @io ||= open
689
+ yield @io
690
+ end
691
+ rescue
692
+ @watcher.log.error $!.to_s
693
+ @watcher.log.error_backtrace
694
+ close
695
+ end
620
696
  end
621
697
  end
622
698
 
@@ -632,44 +708,40 @@ module Fluent::Plugin
632
708
 
633
709
  def close
634
710
  end
711
+
712
+ def opened?
713
+ false
714
+ end
635
715
  end
636
716
 
637
717
  class RotateHandler
638
- def initialize(path, log, &on_rotate)
639
- @path = path
718
+ def initialize(watcher, &on_rotate)
719
+ @watcher = watcher
640
720
  @inode = nil
641
721
  @fsize = -1 # first
642
722
  @on_rotate = on_rotate
643
- @log = log
644
723
  end
645
724
 
646
- def on_notify
647
- begin
648
- stat = Fluent::FileWrapper.stat(@path)
649
- inode = stat.ino
650
- fsize = stat.size
651
- rescue Errno::ENOENT
652
- # moved or deleted
725
+ def on_notify(stat)
726
+ if stat.nil?
653
727
  inode = nil
654
728
  fsize = 0
729
+ else
730
+ inode = stat.ino
731
+ fsize = stat.size
655
732
  end
656
733
 
657
734
  begin
658
735
  if @inode != inode || fsize < @fsize
659
- # rotated or truncated
660
- begin
661
- io = Fluent::FileWrapper.open(@path)
662
- rescue Errno::ENOENT
663
- end
664
- @on_rotate.call(io)
736
+ @on_rotate.call(stat)
665
737
  end
666
738
  @inode = inode
667
739
  @fsize = fsize
668
740
  end
669
741
 
670
742
  rescue
671
- @log.error $!.to_s
672
- @log.error_backtrace
743
+ @watcher.log.error $!.to_s
744
+ @watcher.log.error_backtrace
673
745
  end
674
746
  end
675
747
 
@@ -767,16 +839,19 @@ module Fluent::Plugin
767
839
  def initialize(file, seek)
768
840
  @file = file
769
841
  @seek = seek
842
+ @pos = nil
770
843
  end
771
844
 
772
845
  def update(ino, pos)
773
846
  @file.pos = @seek
774
847
  @file.write "%016x\t%016x" % [pos, ino]
848
+ @pos = pos
775
849
  end
776
850
 
777
851
  def update_pos(pos)
778
852
  @file.pos = @seek
779
853
  @file.write "%016x" % pos
854
+ @pos = pos
780
855
  end
781
856
 
782
857
  def read_inode
@@ -786,9 +861,11 @@ module Fluent::Plugin
786
861
  end
787
862
 
788
863
  def read_pos
789
- @file.pos = @seek
790
- raw = @file.read(16)
791
- raw ? raw.to_i(16) : 0
864
+ @pos ||= begin
865
+ @file.pos = @seek
866
+ raw = @file.read(16)
867
+ raw ? raw.to_i(16) : 0
868
+ end
792
869
  end
793
870
  end
794
871