fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -55,13 +55,12 @@ module Fluent::Plugin
55
55
  def start
56
56
  super
57
57
 
58
- @buffer = ''
59
- server_create(:in_tcp_server, @port, bind: @bind) do |data, conn|
60
- @buffer << data
58
+ server_create(:in_tcp_server, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
59
+ conn.buffer << data
61
60
  begin
62
61
  pos = 0
63
- while i = @buffer.index(@delimiter, pos)
64
- msg = @buffer[pos...i]
62
+ while i = conn.buffer.index(@delimiter, pos)
63
+ msg = conn.buffer[pos...i]
65
64
  pos = i + @delimiter.length
66
65
 
67
66
  @parser.parse(msg) do |time, record|
@@ -77,7 +76,7 @@ module Fluent::Plugin
77
76
  router.emit(tag, time, record)
78
77
  end
79
78
  end
80
- @buffer.slice!(0, pos) if pos > 0
79
+ conn.buffer.slice!(0, pos) if pos > 0
81
80
  end
82
81
  end
83
82
  end
@@ -38,6 +38,10 @@ module Fluent::Plugin
38
38
  config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
39
39
  desc "The max bytes of message"
40
40
  config_param :message_length_limit, :size, default: 4096
41
+ desc "Remove newline from the end of incoming payload"
42
+ config_param :remove_newline, :bool, default: true
43
+ desc "The max size of socket receive buffer. SO_RCVBUF"
44
+ config_param :receive_buffer_size, :size, default: nil
41
45
 
42
46
  config_param :blocking_timeout, :time, default: 0.5
43
47
 
@@ -59,8 +63,8 @@ module Fluent::Plugin
59
63
  super
60
64
 
61
65
  log.info "listening udp socket", bind: @bind, port: @port
62
- server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @message_length_limit) do |data, sock|
63
- data.chomp!
66
+ server_create(:in_udp_server, @port, proto: :udp, bind: @bind, resolve_name: !!@source_hostname_key, max_bytes: @message_length_limit, receive_buffer_size: @receive_buffer_size) do |data, sock|
67
+ data.chomp! if @remove_newline
64
68
  begin
65
69
  @parser.parse(data) do |time, record|
66
70
  unless time && record
@@ -146,7 +146,7 @@ module Fluent
146
146
  @u = Fluent::Engine.msgpack_factory.unpacker
147
147
  end
148
148
 
149
- (class << self; self; end).module_eval do
149
+ singleton_class.module_eval do
150
150
  define_method(:on_read, m)
151
151
  end
152
152
  m.call(data)
@@ -29,6 +29,7 @@ module Fluent
29
29
  helpers :event_emitter # to get router from agent, which will be supplied to child plugins
30
30
 
31
31
  config_section :store, param_name: :stores, multi: true, required: true do
32
+ config_argument :arg, :string, default: ''
32
33
  config_param :@type, :string, default: nil
33
34
  end
34
35
 
@@ -25,6 +25,21 @@ module Fluent::Plugin
25
25
  desc 'If true, pass different record to each `store` plugin.'
26
26
  config_param :deep_copy, :bool, default: false
27
27
 
28
+ attr_reader :ignore_errors
29
+
30
+ def initialize
31
+ super
32
+ @ignore_errors = []
33
+ end
34
+
35
+ def configure(conf)
36
+ super
37
+
38
+ @stores.each { |store|
39
+ @ignore_errors << (store.arg == 'ignore_error')
40
+ }
41
+ end
42
+
28
43
  def multi_workers_ready?
29
44
  true
30
45
  end
@@ -38,8 +53,16 @@ module Fluent::Plugin
38
53
  es = m
39
54
  end
40
55
 
41
- outputs.each do |output|
42
- output.emit_events(tag, @deep_copy ? es.dup : es)
56
+ outputs.each_with_index do |output, i|
57
+ begin
58
+ output.emit_events(tag, @deep_copy ? es.dup : es)
59
+ rescue => e
60
+ if @ignore_errors[i]
61
+ log.error "ignore emit error", error: e
62
+ else
63
+ raise e
64
+ end
65
+ end
43
66
  end
44
67
  end
45
68
  end
@@ -17,6 +17,7 @@
17
17
  require 'fileutils'
18
18
  require 'zlib'
19
19
  require 'time'
20
+ require 'tempfile'
20
21
 
21
22
  require 'fluent/plugin/output'
22
23
  require 'fluent/config/error'
@@ -141,8 +142,8 @@ module Fluent::Plugin
141
142
  dummy_record_keys = get_placeholders_keys(@path_template) || ['message']
142
143
  dummy_record = Hash[dummy_record_keys.zip(['data'] * dummy_record_keys.size)]
143
144
 
144
- test_meta1 = metadata_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
145
- test_path = extract_placeholders(@path_template, test_meta1)
145
+ test_chunk1 = chunk_for_test(dummy_tag, Fluent::Engine.now, dummy_record)
146
+ test_path = extract_placeholders(@path_template, test_chunk1)
146
147
  unless ::Fluent::FileUtil.writable_p?(test_path)
147
148
  raise Fluent::ConfigError, "out_file: `#{test_path}` is not writable"
148
149
  end
@@ -178,7 +179,7 @@ module Fluent::Plugin
178
179
  end
179
180
 
180
181
  def write(chunk)
181
- path = extract_placeholders(@path_template, chunk.metadata)
182
+ path = extract_placeholders(@path_template, chunk)
182
183
  FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
183
184
 
184
185
  writer = case
@@ -213,10 +214,28 @@ module Fluent::Plugin
213
214
  end
214
215
 
215
216
  def write_gzip_with_compression(path, chunk)
216
- File.open(path, "ab", @file_perm) do |f|
217
- gz = Zlib::GzipWriter.new(f)
218
- chunk.write_to(gz, compressed: :text)
219
- gz.close
217
+ if @append
218
+ # This code will be removed after zlib/multithread bug is fixed.
219
+ # Use Tempfile to avoid broken gzip files: https://github.com/fluent/fluentd/issues/1903
220
+ Tempfile.create('out_file-gzip-append') { |temp|
221
+ begin
222
+ writer = Zlib::GzipWriter.new(temp)
223
+ chunk.write_to(writer, compressed: :text)
224
+ ensure
225
+ writer.finish # avoid zlib finalizer warning
226
+ end
227
+ temp.rewind
228
+
229
+ File.open(path, "ab", @file_perm) do |f|
230
+ IO.copy_stream(temp, f)
231
+ end
232
+ }
233
+ else
234
+ File.open(path, "ab", @file_perm) do |f|
235
+ gz = Zlib::GzipWriter.new(f)
236
+ chunk.write_to(gz, compressed: :text)
237
+ gz.close
238
+ end
220
239
  end
221
240
  end
222
241
 
@@ -77,6 +77,9 @@ module Fluent::Plugin
77
77
  desc 'Ignore DNS resolution and errors at startup time.'
78
78
  config_param :ignore_network_errors_at_startup, :bool, default: false
79
79
 
80
+ desc 'Verify that a connection can be made with one of out_forward nodes at the time of startup.'
81
+ config_param :verify_connection_at_startup, :bool, default: false
82
+
80
83
  desc 'Compress buffered data.'
81
84
  config_param :compress, :enum, list: [:text, :gzip], default: :text
82
85
 
@@ -91,6 +94,8 @@ module Fluent::Plugin
91
94
  desc 'Verify hostname of servers and certificates or not in TLS transport.'
92
95
  config_param :tls_verify_hostname, :bool, default: true
93
96
  desc 'The additional CA certificate path for TLS.'
97
+ config_param :tls_ca_cert_path, :array, value_type: :string, default: nil
98
+ desc 'The additional certificate path for TLS.'
94
99
  config_param :tls_cert_path, :array, value_type: :string, default: nil
95
100
 
96
101
  config_section :security, required: false, multi: false do
@@ -166,8 +171,12 @@ module Fluent::Plugin
166
171
  end
167
172
 
168
173
  if @transport == :tls
174
+ # socket helper adds CA cert or signed certificate to same cert store internally so unify it in this place.
169
175
  if @tls_cert_path && !@tls_cert_path.empty?
170
- @tls_cert_path.each do |path|
176
+ @tls_ca_cert_path = @tls_cert_path
177
+ end
178
+ if @tls_ca_cert_path && !@tls_ca_cert_path.empty?
179
+ @tls_ca_cert_path.each do |path|
171
180
  raise Fluent::ConfigError, "specified cert path does not exist:#{path}" unless File.exist?(path)
172
181
  raise Fluent::ConfigError, "specified cert path is not readable:#{path}" unless File.readable?(path)
173
182
  end
@@ -253,6 +262,17 @@ module Fluent::Plugin
253
262
  @sock_ack_waiting = []
254
263
  thread_create(:out_forward_receiving_ack, &method(:ack_reader))
255
264
  end
265
+
266
+ if @verify_connection_at_startup
267
+ @nodes.each do |node|
268
+ begin
269
+ node.verify_connection
270
+ rescue StandardError => e
271
+ log.fatal "forward's connection setting error: #{e.message}"
272
+ raise Fluent::UnrecoverableError, e.message
273
+ end
274
+ end
275
+ end
256
276
  end
257
277
 
258
278
  def close
@@ -324,7 +344,7 @@ module Fluent::Plugin
324
344
  verify_fqdn: @tls_verify_hostname,
325
345
  fqdn: hostname,
326
346
  allow_self_signed_cert: @tls_allow_self_signed_cert,
327
- cert_paths: @tls_cert_path,
347
+ cert_paths: @tls_ca_cert_path,
328
348
  linger_timeout: @send_timeout,
329
349
  send_timeout: @send_timeout,
330
350
  recv_timeout: @ack_response_timeout,
@@ -395,15 +415,19 @@ module Fluent::Plugin
395
415
 
396
416
  def on_timer
397
417
  @nodes.each {|n|
398
- if n.tick
399
- rebuild_weight_array
400
- end
401
418
  begin
402
419
  log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
403
420
  n.usock = @usock if @usock
404
- n.send_heartbeat
405
- rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED
406
- log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: $!
421
+ if n.send_heartbeat
422
+ rebuild_weight_array
423
+ end
424
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
425
+ log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e
426
+ rescue => e
427
+ log.debug "unexpected error happen during heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e
428
+ end
429
+ if n.tick
430
+ rebuild_weight_array
407
431
  end
408
432
  }
409
433
  end
@@ -421,7 +445,7 @@ module Fluent::Plugin
421
445
  def read_ack_from_sock(sock, unpacker)
422
446
  begin
423
447
  raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
424
- rescue Errno::ECONNRESET
448
+ rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
425
449
  raw_data = ""
426
450
  end
427
451
  info = @sock_ack_waiting_mutex.synchronize{ @sock_ack_waiting.find{|i| i.sock == sock } }
@@ -431,16 +455,16 @@ module Fluent::Plugin
431
455
  if raw_data.empty?
432
456
  log.warn "destination node closed the connection. regard it as unavailable.", host: info.node.host, port: info.node.port
433
457
  info.node.disable!
434
- rollback_write(info.chunk_id)
458
+ rollback_write(info.chunk_id, update_retry: false)
435
459
  return nil
436
460
  else
437
461
  unpacker.feed(raw_data)
438
462
  res = unpacker.read
439
463
  log.trace "getting response from destination", host: info.node.host, port: info.node.port, chunk_id: dump_unique_id_hex(info.chunk_id), response: res
440
464
  if res['ack'] != info.chunk_id_base64
441
- # Some errors may have occured when ack and chunk id is different, so send the chunk again.
465
+ # Some errors may have occurred when ack and chunk id is different, so send the chunk again.
442
466
  log.warn "ack in response and chunk id in sent data are different", chunk_id: dump_unique_id_hex(info.chunk_id), ack: res['ack']
443
- rollback_write(info.chunk_id)
467
+ rollback_write(info.chunk_id, update_retry: false)
444
468
  return nil
445
469
  else
446
470
  log.trace "got a correct ack response", chunk_id: dump_unique_id_hex(info.chunk_id)
@@ -481,7 +505,7 @@ module Fluent::Plugin
481
505
  log.warn "no response from node. regard it as unavailable.", host: info.node.host, port: info.node.port
482
506
  info.node.disable!
483
507
  info.sock.close rescue nil
484
- rollback_write(info.chunk_id)
508
+ rollback_write(info.chunk_id, update_retry: false)
485
509
  else
486
510
  sockets << info.sock
487
511
  new_list << info
@@ -517,7 +541,6 @@ module Fluent::Plugin
517
541
  @standby = server.standby
518
542
  @failure = failure
519
543
  @available = true
520
- @state = nil
521
544
 
522
545
  # @hostname is used for certificate verification & TLS SNI
523
546
  host_is_hostname = !(IPAddr.new(@host) rescue false)
@@ -533,7 +556,6 @@ module Fluent::Plugin
533
556
  @password = server.password
534
557
  @shared_key = server.shared_key || (sender.security && sender.security.shared_key) || ""
535
558
  @shared_key_salt = generate_salt
536
- @shared_key_nonce = ""
537
559
 
538
560
  @unpacker = Fluent::Engine.msgpack_unpacker
539
561
 
@@ -548,6 +570,8 @@ module Fluent::Plugin
548
570
  attr_reader :sockaddr # used by on_heartbeat
549
571
  attr_reader :failure, :available # for test
550
572
 
573
+ RequestInfo = Struct.new(:state, :shared_key_nonce, :auth)
574
+
551
575
  def validate_host_resolution!
552
576
  resolved_host
553
577
  end
@@ -564,8 +588,21 @@ module Fluent::Plugin
564
588
  @standby
565
589
  end
566
590
 
567
- def establish_connection(sock)
568
- while available? && @state != :established
591
+ def verify_connection
592
+ sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
593
+ begin
594
+ ri = RequestInfo.new(@sender.security ? :helo : :established)
595
+ if ri.state != :established
596
+ establish_connection(sock, ri)
597
+ raise if ri.state != :established
598
+ end
599
+ ensure
600
+ sock.close
601
+ end
602
+ end
603
+
604
+ def establish_connection(sock, ri)
605
+ while available? && ri.state != :established
569
606
  begin
570
607
  # TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly.
571
608
  # We need rewrite around here using new socket/server plugin helper.
@@ -575,13 +612,13 @@ module Fluent::Plugin
575
612
  next
576
613
  end
577
614
  @unpacker.feed_each(buf) do |data|
578
- on_read(sock, data)
615
+ on_read(sock, ri, data)
579
616
  end
580
617
  rescue IO::WaitReadable
581
618
  # If the exception is Errno::EWOULDBLOCK or Errno::EAGAIN, it is extended by IO::WaitReadable.
582
619
  # So IO::WaitReadable can be used to rescue the exceptions for retrying read_nonblock.
583
- # http://docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock
584
- sleep @sender.read_interval unless @state == :established
620
+ # https//docs.ruby-lang.org/en/2.3.0/IO.html#method-i-read_nonblock
621
+ sleep @sender.read_interval unless ri.state == :established
585
622
  rescue SystemCallError => e
586
623
  @log.warn "disconnected by error", host: @host, port: @port, error: e
587
624
  disable!
@@ -595,9 +632,9 @@ module Fluent::Plugin
595
632
  end
596
633
 
597
634
  def send_data_actual(sock, tag, chunk)
598
- @state = @sender.security ? :helo : :established
599
- if @state != :established
600
- establish_connection(sock)
635
+ ri = RequestInfo.new(@sender.security ? :helo : :established)
636
+ if ri.state != :established
637
+ establish_connection(sock, ri)
601
638
  end
602
639
 
603
640
  unless available?
@@ -663,11 +700,13 @@ module Fluent::Plugin
663
700
  ## don't send any data to not cause a compatibility problem
664
701
  # sock.write FORWARD_TCP_HEARTBEAT_DATA
665
702
 
666
- # successful tcp connection establishment is considered as valid heartbeat
703
+ # successful tcp connection establishment is considered as valid heartbeat.
704
+ # When heartbeat is succeeded after detached, return true. It rebuilds weight array.
667
705
  heartbeat(true)
668
706
  end
669
707
  when :udp
670
708
  @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
709
+ nil
671
710
  when :none # :none doesn't use this class
672
711
  raise "BUG: heartbeat_type none must not use Node"
673
712
  else
@@ -750,7 +789,7 @@ module Fluent::Plugin
750
789
  SecureRandom.hex(16)
751
790
  end
752
791
 
753
- def check_helo(message)
792
+ def check_helo(ri, message)
754
793
  @log.debug "checking helo"
755
794
  # ['HELO', options(hash)]
756
795
  unless message.size == 2 && message[0] == 'HELO'
@@ -758,23 +797,23 @@ module Fluent::Plugin
758
797
  end
759
798
  opts = message[1] || {}
760
799
  # make shared_key_check failed (instead of error) if protocol version mismatch exist
761
- @shared_key_nonce = opts['nonce'] || ''
762
- @authentication = opts['auth'] || ''
800
+ ri.shared_key_nonce = opts['nonce'] || ''
801
+ ri.auth = opts['auth'] || ''
763
802
  true
764
803
  end
765
804
 
766
- def generate_ping
805
+ def generate_ping(ri)
767
806
  @log.debug "generating ping"
768
807
  # ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key),
769
808
  # username || '', sha512\_hex(auth\_salt + username + password) || '']
770
809
  shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt)
771
810
  .update(@sender.security.self_hostname)
772
- .update(@shared_key_nonce)
811
+ .update(ri.shared_key_nonce)
773
812
  .update(@shared_key)
774
813
  .hexdigest
775
814
  ping = ['PING', @sender.security.self_hostname, @shared_key_salt, shared_key_hexdigest]
776
- if !@authentication.empty?
777
- password_hexdigest = Digest::SHA512.new.update(@authentication).update(@username).update(@password).hexdigest
815
+ if !ri.auth.empty?
816
+ password_hexdigest = Digest::SHA512.new.update(ri.auth).update(@username).update(@password).hexdigest
778
817
  ping.push(@username, password_hexdigest)
779
818
  else
780
819
  ping.push('','')
@@ -782,7 +821,7 @@ module Fluent::Plugin
782
821
  ping
783
822
  end
784
823
 
785
- def check_pong(message)
824
+ def check_pong(ri, message)
786
825
  @log.debug "checking pong"
787
826
  # ['PONG', bool(authentication result), 'reason if authentication failed',
788
827
  # self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)]
@@ -799,7 +838,7 @@ module Fluent::Plugin
799
838
  return false, 'same hostname between input and output: invalid configuration'
800
839
  end
801
840
 
802
- clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(@shared_key_nonce).update(@shared_key).hexdigest
841
+ clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(ri.shared_key_nonce).update(@shared_key).hexdigest
803
842
  unless shared_key_hexdigest == clientside
804
843
  return false, 'shared key mismatch'
805
844
  end
@@ -807,29 +846,29 @@ module Fluent::Plugin
807
846
  return true, nil
808
847
  end
809
848
 
810
- def on_read(sock, data)
849
+ def on_read(sock, ri, data)
811
850
  @log.trace __callee__
812
851
 
813
- case @state
852
+ case ri.state
814
853
  when :helo
815
- unless check_helo(data)
854
+ unless check_helo(ri, data)
816
855
  @log.warn "received invalid helo message from #{@name}"
817
856
  disable! # shutdown
818
857
  return
819
858
  end
820
- sock.write(generate_ping.to_msgpack)
821
- @state = :pingpong
859
+ sock.write(generate_ping(ri).to_msgpack)
860
+ ri.state = :pingpong
822
861
  when :pingpong
823
- succeeded, reason = check_pong(data)
862
+ succeeded, reason = check_pong(ri, data)
824
863
  unless succeeded
825
- @log.warn "connection refused to #{@name}: #{reason}"
864
+ @log.warn "connection refused to #{@name || @host}: #{reason}"
826
865
  disable! # shutdown
827
866
  return
828
867
  end
829
- @state = :established
868
+ ri.state = :established
830
869
  @log.debug "connection established", host: @host, port: @port
831
870
  else
832
- raise "BUG: unknown session state: #{@state}"
871
+ raise "BUG: unknown session state: #{ri.state}"
833
872
  end
834
873
  end
835
874
  end