fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -165,18 +165,26 @@ module Fluent
165
165
  end
166
166
 
167
167
  def calc_interval(num)
168
- # make it infinite if calculated "interval" is too big
169
- interval = @constant_factor.to_f * (@backoff_base ** (num - 1))
170
- if interval.finite?
171
- if @max_interval && interval > @max_interval
172
- @max_interval
168
+ interval = raw_interval(num - 1)
169
+ if @max_interval && interval > @max_interval
170
+ @max_interval
171
+ else
172
+ if interval.finite?
173
+ interval
173
174
  else
175
+ # Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.
176
+ until interval.finite?
177
+ num -= 1
178
+ interval = raw_interval(num - 1)
179
+ end
174
180
  interval
175
181
  end
176
- else
177
- interval
178
182
  end
179
183
  end
184
+
185
+ def raw_interval(num)
186
+ @constant_factor.to_f * (@backoff_base ** (num))
187
+ end
180
188
  end
181
189
 
182
190
  class PeriodicRetry < RetryStateMachine
@@ -213,8 +213,10 @@ module Fluent
213
213
  socket_option_setter.call(sock)
214
214
  close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
215
215
  server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
216
- @_server_mutex.synchronize do
217
- @_server_connections << conn
216
+ unless conn.closing
217
+ @_server_mutex.synchronize do
218
+ @_server_connections << conn
219
+ end
218
220
  end
219
221
  end
220
222
  server.listen(backlog) if backlog
@@ -227,8 +229,10 @@ module Fluent
227
229
  socket_option_setter.call(sock)
228
230
  close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
229
231
  server = Coolio::TCPServer.new(sock, nil, EventHandler::TLSServer, context, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
230
- @_server_mutex.synchronize do
231
- @_server_connections << conn
232
+ unless conn.closing
233
+ @_server_mutex.synchronize do
234
+ @_server_connections << conn
235
+ end
232
236
  end
233
237
  end
234
238
  server.listen(backlog) if backlog
@@ -237,7 +241,7 @@ module Fluent
237
241
 
238
242
  SERVER_TRANSPORT_PARAMS = [
239
243
  :protocol, :version, :ciphers, :insecure,
240
- :cert_path, :private_key_path, :private_key_passphrase,
244
+ :ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,
241
245
  :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
242
246
  :generate_private_key_length,
243
247
  :generate_cert_country, :generate_cert_state, :generate_cert_state,
@@ -271,9 +275,11 @@ module Fluent
271
275
  config_param :insecure, :bool, default: false
272
276
 
273
277
  # Cert signed by public CA
278
+ config_param :ca_path, :string, default: nil
274
279
  config_param :cert_path, :string, default: nil
275
280
  config_param :private_key_path, :string, default: nil
276
281
  config_param :private_key_passphrase, :string, default: nil, secret: true
282
+ config_param :client_cert_auth, :bool, default: false
277
283
 
278
284
  # Cert generated and signed by private CA Certificate
279
285
  config_param :ca_cert_path, :string, default: nil
@@ -370,6 +376,9 @@ module Fluent
370
376
  sock
371
377
  end
372
378
 
379
+ # Use string "?" for port, not integer or nil. "?" is clear than -1 or nil in the log.
380
+ PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
381
+
373
382
  class CallbackSocket
374
383
  def initialize(server_type, sock, enabled_events = [], close_socket: true)
375
384
  @server_type = server_type
@@ -399,6 +408,10 @@ module Fluent
399
408
  raise "not implemented here"
400
409
  end
401
410
 
411
+ def close_after_write_complete
412
+ @sock.close_after_write_complete = true
413
+ end
414
+
402
415
  def close
403
416
  @sock.close if @close_socket
404
417
  end
@@ -425,9 +438,14 @@ module Fluent
425
438
  end
426
439
 
427
440
  class TCPCallbackSocket < CallbackSocket
441
+ ENABLED_EVENTS = [:data, :write_complete, :close]
442
+
443
+ attr_accessor :buffer
444
+
428
445
  def initialize(sock)
429
- super("tcp", sock, [:data, :write_complete, :close])
430
- @peeraddr = @sock.peeraddr
446
+ super("tcp", sock, ENABLED_EVENTS)
447
+ @peeraddr = (@sock.peeraddr rescue PEERADDR_FAILED)
448
+ @buffer = ''
431
449
  end
432
450
 
433
451
  def write(data)
@@ -436,9 +454,14 @@ module Fluent
436
454
  end
437
455
 
438
456
  class TLSCallbackSocket < CallbackSocket
457
+ ENABLED_EVENTS = [:data, :write_complete, :close]
458
+
459
+ attr_accessor :buffer
460
+
439
461
  def initialize(sock)
440
- super("tls", sock, [:data, :write_complete, :close])
441
- @peeraddr = @sock.to_io.peeraddr
462
+ super("tls", sock, ENABLED_EVENTS)
463
+ @peeraddr = (@sock.to_io.peeraddr rescue PEERADDR_FAILED)
464
+ @buffer = ''
442
465
  end
443
466
 
444
467
  def write(data)
@@ -447,8 +470,10 @@ module Fluent
447
470
  end
448
471
 
449
472
  class UDPCallbackSocket < CallbackSocket
473
+ ENABLED_EVENTS = []
474
+
450
475
  def initialize(sock, peeraddr, **kwargs)
451
- super("udp", sock, [], **kwargs)
476
+ super("udp", sock, ENABLED_EVENTS, **kwargs)
452
477
  @peeraddr = peeraddr
453
478
  end
454
479
 
@@ -471,6 +496,8 @@ module Fluent
471
496
 
472
497
  module EventHandler
473
498
  class UDPServer < Coolio::IO
499
+ attr_writer :close_after_write_complete # dummy for consistent method call in callbacks
500
+
474
501
  def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
475
502
  raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
476
503
 
@@ -521,6 +548,9 @@ module Fluent
521
548
  end
522
549
 
523
550
  class TCPServer < Coolio::TCPSocket
551
+ attr_reader :closing
552
+ attr_writer :close_after_write_complete
553
+
524
554
  def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
525
555
  raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
526
556
 
@@ -537,6 +567,7 @@ module Fluent
537
567
  @close_callback = close_callback
538
568
 
539
569
  @callback_connection = nil
570
+ @close_after_write_complete = false
540
571
  @closing = false
541
572
 
542
573
  @mutex = Mutex.new # to serialize #write and #close
@@ -564,6 +595,11 @@ module Fluent
564
595
  end
565
596
  end
566
597
 
598
+ def on_writable
599
+ super
600
+ close if @close_after_write_complete
601
+ end
602
+
567
603
  def on_connect
568
604
  @callback_connection = TCPCallbackSocket.new(self)
569
605
  @connect_callback.call(@callback_connection)
@@ -577,7 +613,7 @@ module Fluent
577
613
  rescue => e
578
614
  @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
579
615
  @log.error_backtrace
580
- close(true) rescue nil
616
+ close rescue nil
581
617
  raise if @under_plugin_development
582
618
  end
583
619
 
@@ -586,7 +622,7 @@ module Fluent
586
622
  rescue => e
587
623
  @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
588
624
  @log.error_backtrace
589
- close(true) rescue nil
625
+ close rescue nil
590
626
  raise if @under_plugin_development
591
627
  end
592
628
 
@@ -601,6 +637,9 @@ module Fluent
601
637
  end
602
638
 
603
639
  class TLSServer < Coolio::Socket
640
+ attr_reader :closing
641
+ attr_writer :close_after_write_complete
642
+
604
643
  # It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.
605
644
  def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
606
645
  raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
@@ -620,6 +659,7 @@ module Fluent
620
659
  @close_callback = close_callback
621
660
 
622
661
  @callback_connection = nil
662
+ @close_after_write_complete = false
623
663
  @closing = false
624
664
 
625
665
  @mutex = Mutex.new # to serialize #write and #close
@@ -649,22 +689,39 @@ module Fluent
649
689
  end
650
690
  end
651
691
 
692
+ if RUBY_VERSION.to_f >= 2.3
693
+ NONBLOCK_ARG = {exception: false}
694
+ def try_handshake
695
+ @_handler_socket.accept_nonblock(NONBLOCK_ARG)
696
+ end
697
+ else
698
+ def try_handshake
699
+ @_handler_socket.accept_nonblock
700
+ rescue IO::WaitReadable
701
+ :wait_readable
702
+ rescue IO::WaitWritable
703
+ :wait_writable
704
+ end
705
+ end
706
+
652
707
  def try_tls_accept
653
708
  return true if @_handler_accepted
654
709
 
655
710
  begin
656
- @_handler_socket.accept_nonblock # this method call actually try to do handshake via TLS
657
- @_handler_accepted = true
658
-
659
- @callback_connection = TLSCallbackSocket.new(self)
660
- @connect_callback.call(@callback_connection)
661
- unless @data_callback
662
- raise "connection callback must call #data to set data callback"
711
+ result = try_handshake # this method call actually try to do handshake via TLS
712
+ if result == :wait_readable || result == :wait_writable
713
+ # retry accept_nonblock: there aren't enough data in underlying socket buffer
714
+ else
715
+ @_handler_accepted = true
716
+
717
+ @callback_connection = TLSCallbackSocket.new(self)
718
+ @connect_callback.call(@callback_connection)
719
+ unless @data_callback
720
+ raise "connection callback must call #data to set data callback"
721
+ end
722
+
723
+ return true
663
724
  end
664
- return true
665
-
666
- rescue IO::WaitReadable, IO::WaitWritable
667
- # retry accept_nonblock: there aren't enough data in underlying socket buffer
668
725
  rescue OpenSSL::SSL::SSLError => e
669
726
  @log.trace "unexpected error before accepting TLS connection", error: e
670
727
  close rescue nil
@@ -683,15 +740,20 @@ module Fluent
683
740
  end
684
741
  rescue IO::WaitReadable, IO::WaitWritable
685
742
  # ignore and return with doing nothing
743
+ rescue OpenSSL::SSL::SSLError => e
744
+ @log.warn "close socket due to unexpected ssl error: #{e}"
745
+ close rescue nil
686
746
  end
687
747
 
688
748
  def on_writable
689
749
  begin
690
750
  @mutex.synchronize do
751
+ # Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.
691
752
  written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
692
753
  @_handler_write_buffer.slice!(0, written_bytes)
693
754
  super
694
755
  end
756
+ close if @close_after_write_complete
695
757
  rescue IO::WaitWritable, IO::WaitReadable
696
758
  return
697
759
  rescue Errno::EINTR
@@ -710,7 +772,7 @@ module Fluent
710
772
  rescue => e
711
773
  @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
712
774
  @log.error_backtrace
713
- close(true) rescue nil
775
+ close rescue nil
714
776
  raise if @under_plugin_development
715
777
  end
716
778
 
@@ -719,7 +781,7 @@ module Fluent
719
781
  rescue => e
720
782
  @log.error "unexpected error on reading data", host: @callback_connection.remote_host, port: @callback_connection.remote_port, error: e
721
783
  @log.error_backtrace
722
- close(true) rescue nil
784
+ close rescue nil
723
785
  raise if @under_plugin_development
724
786
  end
725
787
 
@@ -24,7 +24,7 @@ module Fluent
24
24
  FORMAT_STRUCT_LINGER = 'I!I!' # { int l_onoff; int l_linger; }
25
25
  FORMAT_STRUCT_TIMEVAL = 'L!L!' # { time_t tv_sec; suseconds_t tv_usec; }
26
26
 
27
- def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil)
27
+ def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, receive_buffer_size: nil)
28
28
  unless resolve_name.nil?
29
29
  if protocol != :tcp && protocol != :udp && protocol != :tls
30
30
  raise ArgumentError, "BUG: resolve_name in available for tcp/udp/tls"
@@ -37,7 +37,7 @@ module Fluent
37
37
  end
38
38
  end
39
39
 
40
- def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil)
40
+ def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, receive_buffer_size: nil)
41
41
  unless resolve_name.nil?
42
42
  sock.do_not_reverse_lookup = !resolve_name
43
43
  end
@@ -56,6 +56,9 @@ module Fluent
56
56
  optval = [send_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)
57
57
  socket_option_set_one(sock, :SO_SNDTIMEO, optval)
58
58
  end
59
+ if receive_buffer_size
60
+ socket_option_set_one(sock, :SO_RCVBUF, receive_buffer_size.to_i)
61
+ end
59
62
  sock
60
63
  end
61
64
 
@@ -15,6 +15,7 @@
15
15
  #
16
16
 
17
17
  require 'fluent/plugin_helper/event_loop'
18
+ require 'set'
18
19
 
19
20
  module Fluent
20
21
  module PluginHelper
@@ -33,7 +34,8 @@ module Fluent
33
34
  raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
34
35
  raise ArgumentError, "BUG: block not specified for callback" unless block_given?
35
36
  checker = ->(){ @_timer_running }
36
- timer = TimerWatcher.new(title, interval, repeat, log, checker, &block)
37
+ detacher = ->(watcher){ event_loop_detach(watcher) }
38
+ timer = TimerWatcher.new(title, interval, repeat, log, checker, detacher, &block)
37
39
  @_timers << title
38
40
  event_loop_attach(timer)
39
41
  timer
@@ -45,7 +47,7 @@ module Fluent
45
47
 
46
48
  def initialize
47
49
  super
48
- @_timers = []
50
+ @_timers ||= Set.new
49
51
  end
50
52
 
51
53
  def start
@@ -60,16 +62,17 @@ module Fluent
60
62
 
61
63
  def terminate
62
64
  super
63
- @_timers = []
65
+ @_timers = nil
64
66
  end
65
67
 
66
68
  class TimerWatcher < Coolio::TimerWatcher
67
- def initialize(title, interval, repeat, log, checker, &callback)
69
+ def initialize(title, interval, repeat, log, checker, detacher, &callback)
68
70
  @title = title
69
71
  @callback = callback
70
72
  @repeat = repeat
71
73
  @log = log
72
74
  @checker = checker
75
+ @detacher = detacher
73
76
  super(interval, repeat)
74
77
  end
75
78
 
@@ -81,9 +84,7 @@ module Fluent
81
84
  detach
82
85
  @log.error "Timer detached.", title: @title
83
86
  ensure
84
- if attached?
85
- detach unless @repeat
86
- end
87
+ @detacher.call(self) unless @repeat
87
88
  end
88
89
  end
89
90
  end
@@ -212,7 +212,6 @@ module Fluent
212
212
 
213
213
  lifecycle_unsafe_sequence = ->(method, checker) {
214
214
  operation = case method
215
- when :before_shutdown then "preparing shutdown"
216
215
  when :shutdown then "shutting down"
217
216
  when :close then "closing"
218
217
  else
@@ -228,11 +227,23 @@ module Fluent
228
227
  Thread.current.abort_on_exception = true
229
228
  begin
230
229
  if method == :shutdown
230
+ # To avoid Input#shutdown and Output#before_shutdown mismatch problem, combine before_shutdown and shutdown call in one sequence.
231
+ # The problem is in_tail flushes buffered multiline in shutdown but output's flush_at_shutdown is invoked in before_shutdown
232
+ operation = "preparing shutdown" # for logging
233
+ log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
234
+ begin
235
+ instance.send(:before_shutdown) unless instance.send(:before_shutdown?)
236
+ rescue Exception => e
237
+ log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
238
+ log.warn_backtrace
239
+ end
240
+ operation = "shutting down"
231
241
  log.info "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
242
+ instance.send(:shutdown) unless instance.send(:shutdown?)
232
243
  else
233
244
  log.debug "#{operation} #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
245
+ instance.send(method) unless instance.send(checker)
234
246
  end
235
- instance.send(method) unless instance.send(checker)
236
247
  rescue Exception => e
237
248
  log.warn "unexpected error while #{operation} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
238
249
  log.warn_backtrace
@@ -245,8 +256,6 @@ module Fluent
245
256
  lifecycle_safe_sequence.call(:stop, :stopped?)
246
257
 
247
258
  # before_shutdown does force_flush for output plugins: it should block, so it's unsafe operation
248
- lifecycle_unsafe_sequence.call(:before_shutdown, :before_shutdown?)
249
-
250
259
  lifecycle_unsafe_sequence.call(:shutdown, :shutdown?)
251
260
 
252
261
  lifecycle_safe_sequence.call(:after_shutdown, :after_shutdown?)
@@ -292,7 +301,7 @@ module Fluent
292
301
  end
293
302
 
294
303
  def emit_error_event(tag, time, record, error)
295
- error_info = {error: error, tag: tag, time: time}
304
+ error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}
296
305
  if @error_collector
297
306
  # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
298
307
  log.warn "send an error event to @ERROR:", error_info
@@ -304,12 +313,12 @@ module Fluent
304
313
  end
305
314
 
306
315
  def handle_emits_error(tag, es, error)
307
- error_info = {error: error, tag: tag}
316
+ error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
308
317
  if @error_collector
309
318
  log.warn "send an error event stream to @ERROR:", error_info
310
319
  @error_collector.emit_stream(tag, es)
311
320
  else
312
- now = Time.now
321
+ now = Time.now.to_i
313
322
  if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
314
323
  log.warn "emit transaction failed:", error_info
315
324
  log.warn_backtrace
@@ -333,12 +342,12 @@ module Fluent
333
342
  end
334
343
 
335
344
  def emit_error_event(tag, time, record, error)
336
- error_info = {error: error, tag: tag, time: time, record: record}
345
+ error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
337
346
  log.warn "dump an error event in @ERROR:", error_info
338
347
  end
339
348
 
340
349
  def handle_emits_error(tag, es, e)
341
- now = EventTime.now
350
+ now = EventTime.now.to_i
342
351
  if @suppress_emit_error_log_interval.zero? || now > @next_emit_error_log_time
343
352
  log.warn "emit transaction failed in @ERROR:", error: e, tag: tag
344
353
  log.warn_backtrace