fluentd 0.14.11-x86-mingw32 → 0.14.12-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -5
  3. data/ChangeLog +54 -2
  4. data/example/in_dummy_blocks.conf +17 -0
  5. data/example/in_forward_tls.conf +14 -0
  6. data/example/in_forward_workers.conf +21 -0
  7. data/example/logevents.conf +25 -0
  8. data/example/out_forward_heartbeat_none.conf +16 -0
  9. data/example/out_forward_tls.conf +18 -0
  10. data/example/suppress_config_dump.conf +7 -0
  11. data/lib/fluent/agent.rb +3 -32
  12. data/lib/fluent/clock.rb +62 -0
  13. data/lib/fluent/command/fluentd.rb +12 -0
  14. data/lib/fluent/compat/input.rb +10 -1
  15. data/lib/fluent/compat/output.rb +40 -1
  16. data/lib/fluent/config/configure_proxy.rb +30 -7
  17. data/lib/fluent/config/section.rb +4 -0
  18. data/lib/fluent/config/types.rb +2 -2
  19. data/lib/fluent/configurable.rb +31 -5
  20. data/lib/fluent/engine.rb +61 -12
  21. data/lib/fluent/event_router.rb +6 -0
  22. data/lib/fluent/load.rb +0 -1
  23. data/lib/fluent/log.rb +118 -42
  24. data/lib/fluent/match.rb +37 -0
  25. data/lib/fluent/plugin.rb +25 -3
  26. data/lib/fluent/plugin/base.rb +4 -0
  27. data/lib/fluent/plugin/buf_file.rb +38 -14
  28. data/lib/fluent/plugin/buffer.rb +20 -20
  29. data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
  30. data/lib/fluent/plugin/compressable.rb +1 -0
  31. data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
  32. data/lib/fluent/plugin/formatter_csv.rb +4 -1
  33. data/lib/fluent/plugin/formatter_hash.rb +5 -1
  34. data/lib/fluent/plugin/formatter_json.rb +10 -0
  35. data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
  36. data/lib/fluent/plugin/in_dummy.rb +4 -0
  37. data/lib/fluent/plugin/in_exec.rb +4 -0
  38. data/lib/fluent/plugin/in_forward.rb +11 -3
  39. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  40. data/lib/fluent/plugin/in_http.rb +4 -0
  41. data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
  42. data/lib/fluent/plugin/in_object_space.rb +4 -1
  43. data/lib/fluent/plugin/in_syslog.rb +4 -0
  44. data/lib/fluent/plugin/in_tail.rb +193 -116
  45. data/lib/fluent/plugin/in_tcp.rb +5 -1
  46. data/lib/fluent/plugin/in_udp.rb +4 -0
  47. data/lib/fluent/plugin/input.rb +4 -0
  48. data/lib/fluent/plugin/out_copy.rb +4 -0
  49. data/lib/fluent/plugin/out_exec.rb +4 -0
  50. data/lib/fluent/plugin/out_exec_filter.rb +4 -0
  51. data/lib/fluent/plugin/out_file.rb +70 -30
  52. data/lib/fluent/plugin/out_forward.rb +132 -28
  53. data/lib/fluent/plugin/out_null.rb +10 -0
  54. data/lib/fluent/plugin/out_relabel.rb +4 -0
  55. data/lib/fluent/plugin/out_roundrobin.rb +4 -0
  56. data/lib/fluent/plugin/out_secondary_file.rb +5 -0
  57. data/lib/fluent/plugin/out_stdout.rb +5 -0
  58. data/lib/fluent/plugin/output.rb +18 -9
  59. data/lib/fluent/plugin/storage_local.rb +25 -2
  60. data/lib/fluent/plugin_helper/cert_option.rb +159 -0
  61. data/lib/fluent/plugin_helper/child_process.rb +6 -6
  62. data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
  63. data/lib/fluent/plugin_helper/event_loop.rb +29 -4
  64. data/lib/fluent/plugin_helper/inject.rb +14 -1
  65. data/lib/fluent/plugin_helper/server.rb +275 -31
  66. data/lib/fluent/plugin_helper/socket.rb +144 -4
  67. data/lib/fluent/plugin_helper/socket_option.rb +2 -17
  68. data/lib/fluent/plugin_helper/storage.rb +7 -1
  69. data/lib/fluent/plugin_helper/thread.rb +16 -4
  70. data/lib/fluent/registry.rb +26 -9
  71. data/lib/fluent/root_agent.rb +7 -3
  72. data/lib/fluent/supervisor.rb +37 -15
  73. data/lib/fluent/system_config.rb +37 -10
  74. data/lib/fluent/test.rb +2 -0
  75. data/lib/fluent/test/driver/base.rb +24 -26
  76. data/lib/fluent/test/helpers.rb +21 -0
  77. data/lib/fluent/version.rb +1 -1
  78. data/test/command/test_fluentd.rb +274 -4
  79. data/test/config/test_configurable.rb +154 -0
  80. data/test/config/test_configure_proxy.rb +180 -1
  81. data/test/config/test_system_config.rb +10 -0
  82. data/test/config/test_types.rb +1 -0
  83. data/test/plugin/test_base.rb +4 -0
  84. data/test/plugin/test_buf_file.rb +241 -9
  85. data/test/plugin/test_buffer.rb +11 -11
  86. data/test/plugin/test_buffer_file_chunk.rb +6 -6
  87. data/test/plugin/test_compressable.rb +3 -0
  88. data/test/plugin/test_filter.rb +4 -0
  89. data/test/plugin/test_filter_record_transformer.rb +20 -0
  90. data/test/plugin/test_formatter_csv.rb +9 -0
  91. data/test/plugin/test_formatter_hash.rb +35 -0
  92. data/test/plugin/test_formatter_json.rb +8 -0
  93. data/test/plugin/test_formatter_ltsv.rb +7 -0
  94. data/test/plugin/test_in_dummy.rb +7 -3
  95. data/test/plugin/test_in_monitor_agent.rb +43 -5
  96. data/test/plugin/test_in_tail.rb +97 -4
  97. data/test/plugin/test_input.rb +4 -0
  98. data/test/plugin/test_out_file.rb +46 -7
  99. data/test/plugin/test_out_forward.rb +59 -7
  100. data/test/plugin/test_output.rb +10 -4
  101. data/test/plugin/test_output_as_buffered.rb +37 -25
  102. data/test/plugin/test_output_as_buffered_compress.rb +1 -1
  103. data/test/plugin/test_output_as_buffered_retries.rb +6 -6
  104. data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
  105. data/test/plugin/test_storage_local.rb +40 -1
  106. data/test/plugin_helper/test_child_process.rb +29 -28
  107. data/test/plugin_helper/test_compat_parameters.rb +1 -1
  108. data/test/plugin_helper/test_inject.rb +27 -9
  109. data/test/plugin_helper/test_server.rb +822 -50
  110. data/test/plugin_helper/test_storage.rb +11 -0
  111. data/test/plugin_helper/test_timer.rb +1 -0
  112. data/test/test_clock.rb +164 -0
  113. data/test/test_log.rb +146 -15
  114. data/test/test_plugin.rb +251 -0
  115. data/test/test_supervisor.rb +65 -57
  116. data/test/test_test_drivers.rb +2 -2
  117. metadata +18 -7
  118. data/lib/fluent/process.rb +0 -504
  119. data/test/test_process.rb +0 -48
@@ -48,11 +48,15 @@ module Fluent::Plugin
48
48
  @parser = parser_create
49
49
  end
50
50
 
51
+ def multi_workers_ready?
52
+ true
53
+ end
54
+
51
55
  def start
52
56
  super
53
57
 
54
58
  @buffer = ''
55
- server_create(:in_tcp_server, @port, proto: :tcp, bind: @bind) do |data, conn|
59
+ server_create(:in_tcp_server, @port, bind: @bind) do |data, conn|
56
60
  @buffer << data
57
61
  begin
58
62
  pos = 0
@@ -47,6 +47,10 @@ module Fluent::Plugin
47
47
  @parser = parser_create
48
48
  end
49
49
 
50
+ def multi_workers_ready?
51
+ true
52
+ end
53
+
50
54
  def start
51
55
  super
52
56
 
@@ -28,6 +28,10 @@ module Fluent
28
28
  include PluginHelper::Mixin
29
29
 
30
30
  helpers :event_emitter
31
+
32
+ def multi_workers_ready?
33
+ false
34
+ end
31
35
  end
32
36
  end
33
37
  end
@@ -25,6 +25,10 @@ module Fluent::Plugin
25
25
  desc 'If true, pass different record to each `store` plugin.'
26
26
  config_param :deep_copy, :bool, default: false
27
27
 
28
+ def multi_workers_ready?
29
+ true
30
+ end
31
+
28
32
  def process(tag, es)
29
33
  unless es.repeatable?
30
34
  m = Fluent::MultiEventStream.new
@@ -51,6 +51,10 @@ module Fluent::Plugin
51
51
  @formatter = formatter_create
52
52
  end
53
53
 
54
+ def multi_workers_ready?
55
+ true
56
+ end
57
+
54
58
  NEWLINE = "\n"
55
59
 
56
60
  def format(tag, time, record)
@@ -172,6 +172,10 @@ module Fluent::Plugin
172
172
  @next_log_time = Time.now.to_i
173
173
  end
174
174
 
175
+ def multi_workers_ready?
176
+ true
177
+ end
178
+
175
179
  ExecutedProcess = Struct.new(:mutex, :pid, :respawns, :readio, :writeio)
176
180
 
177
181
  def start
@@ -93,14 +93,15 @@ module Fluent::Plugin
93
93
 
94
94
  configured_time_slice_format = conf['time_slice_format']
95
95
 
96
- # v0.14 file buffer handles path as directory if '*' is missing
97
- # 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
98
- # but raise it in this plugin.
99
96
  if conf.elements(name: 'buffer').empty?
100
97
  conf.add_element('buffer', 'time')
101
98
  end
102
99
  buffer_conf = conf.elements(name: 'buffer').first
103
- unless buffer_conf.has_key?('path')
100
+ # Fluent::PluginId#configure is not called yet, so we can't use #plugin_root_dir here.
101
+ if !buffer_conf.has_key?('path') && !(conf['@id'] && system_config.root_dir)
102
+ # v0.14 file buffer handles path as directory if '*' is missing
103
+ # 'dummy_path' is not to raise configuration error for 'path' in file buffer plugin,
104
+ # but raise it in this plugin.
104
105
  buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
105
106
  end
106
107
 
@@ -159,6 +160,11 @@ module Fluent::Plugin
159
160
 
160
161
  @dir_perm = system_config.dir_permission || DIR_PERMISSION
161
162
  @file_perm = system_config.file_permission || FILE_PERMISSION
163
+ @need_lock = system_config.workers > 1
164
+ end
165
+
166
+ def multi_workers_ready?
167
+ true
162
168
  end
163
169
 
164
170
  def format(tag, time, record)
@@ -170,34 +176,51 @@ module Fluent::Plugin
170
176
  path = extract_placeholders(@path_template, chunk.metadata)
171
177
  FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
172
178
 
173
- unless @append
174
- path = find_filepath_available(path)
175
- end
176
-
177
- case @compress_method
178
- when nil
179
- File.open(path, "ab", @file_perm) do |f|
180
- chunk.write_to(f)
181
- end
182
- when :gzip
183
- if @buffer.compress != :gzip || @recompress
184
- File.open(path, "ab", @file_perm) do |f|
185
- gz = Zlib::GzipWriter.new(f)
186
- chunk.write_to(gz, compressed: :text)
187
- gz.close
188
- end
189
- else
190
- File.open(path, "ab", @file_perm) do |f|
191
- chunk.write_to(f, compressed: :gzip)
192
- end
193
- end
179
+ writer = case
180
+ when @compress_method.nil?
181
+ method(:write_without_compression)
182
+ when @compress_method == :gzip
183
+ if @buffer.compress != :gzip || @recompress
184
+ method(:write_gzip_with_compression)
185
+ else
186
+ method(:write_gzip_from_gzipped_chunk)
187
+ end
188
+ else
189
+ raise "BUG: unknown compression method #{@compress_method}"
190
+ end
191
+
192
+ if @append
193
+ writer.call(path, chunk)
194
194
  else
195
- raise "BUG: unknown compression method #{@compress_method}"
195
+ find_filepath_available(path, with_lock: @need_lock) do |actual_path|
196
+ writer.call(actual_path, chunk)
197
+ path = actual_path
198
+ end
196
199
  end
197
200
 
198
201
  @last_written_path = path
199
202
  end
200
203
 
204
+ def write_without_compression(path, chunk)
205
+ File.open(path, "ab", @file_perm) do |f|
206
+ chunk.write_to(f)
207
+ end
208
+ end
209
+
210
+ def write_gzip_with_compression(path, chunk)
211
+ File.open(path, "ab", @file_perm) do |f|
212
+ gz = Zlib::GzipWriter.new(f)
213
+ chunk.write_to(gz, compressed: :text)
214
+ gz.close
215
+ end
216
+ end
217
+
218
+ def write_gzip_from_gzipped_chunk(path, chunk)
219
+ File.open(path, "ab", @file_perm) do |f|
220
+ chunk.write_to(f, compressed: :gzip)
221
+ end
222
+ end
223
+
201
224
  def timekey_to_timeformat(timekey)
202
225
  case timekey
203
226
  when nil then ''
@@ -249,14 +272,31 @@ module Fluent::Plugin
249
272
  end
250
273
  end
251
274
 
252
- def find_filepath_available(path_with_placeholder) # for non-append
275
+ def find_filepath_available(path_with_placeholder, with_lock: false) # for non-append
253
276
  raise "BUG: index placeholder not found in path: #{path_with_placeholder}" unless path_with_placeholder.index('_**')
254
277
  i = 0
255
- while path = path_with_placeholder.sub('_**', "_#{i}")
256
- break unless File.exist?(path)
278
+ dir_path = locked = nil
279
+ while true
280
+ path = path_with_placeholder.sub('_**', "_#{i}")
257
281
  i += 1
282
+ next if File.exist?(path)
283
+
284
+ if with_lock
285
+ dir_path = path + '.lock'
286
+ locked = Dir.mkdir(dir_path) rescue false
287
+ next unless locked
288
+ # ensure that other worker doesn't create a file (and release lock)
289
+ # between previous File.exist? and Dir.mkdir
290
+ next if File.exist?(path)
291
+ end
292
+
293
+ break
294
+ end
295
+ yield path
296
+ ensure
297
+ if dir_path && locked && Dir.exist?(dir_path)
298
+ Dir.rmdir(dir_path) rescue nil
258
299
  end
259
- path
260
300
  end
261
301
  end
262
302
  end
@@ -16,6 +16,7 @@
16
16
 
17
17
  require 'fluent/output'
18
18
  require 'fluent/config/error'
19
+ require 'fluent/clock'
19
20
  require 'base64'
20
21
 
21
22
  require 'fluent/compat/socket_util'
@@ -32,14 +33,17 @@ module Fluent::Plugin
32
33
 
33
34
  LISTEN_PORT = 24224
34
35
 
35
- PROCESS_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
36
+ desc 'The transport protocol.'
37
+ config_param :transport, :enum, list: [:tcp, :tls], default: :tcp
38
+ # TODO: TLS session cache/tickets
39
+ # TODO: Connection keepalive
36
40
 
37
41
  desc 'The timeout time when sending event logs.'
38
42
  config_param :send_timeout, :time, default: 60
39
43
  # TODO: add linger_timeout, recv_timeout
40
44
 
41
- desc 'The transport protocol to use for heartbeats.(udp,tcp,none)'
42
- config_param :heartbeat_type, :enum, list: [:tcp, :udp, :none], default: :tcp
45
+ desc 'The protocol to use for heartbeats (default is the same with "transport").'
46
+ config_param :heartbeat_type, :enum, list: [:transport, :tcp, :udp, :none], default: :transport
43
47
  desc 'The interval of the heartbeat packer.'
44
48
  config_param :heartbeat_interval, :time, default: 1
45
49
  desc 'The wait time before accepting a server fault recovery.'
@@ -70,9 +74,25 @@ module Fluent::Plugin
70
74
  desc 'Enable client-side DNS round robin.'
71
75
  config_param :dns_round_robin, :bool, default: false # heartbeat_type 'udp' is not available for this
72
76
 
77
+ desc 'Ignore DNS resolution and errors at startup time.'
78
+ config_param :ignore_network_errors_at_startup, :bool, default: false
79
+
73
80
  desc 'Compress buffered data.'
74
81
  config_param :compress, :enum, list: [:text, :gzip], default: :text
75
82
 
83
+ desc 'The default version of TLS transport.'
84
+ config_param :tls_version, :enum, list: Fluent::PluginHelper::Socket::TLS_SUPPORTED_VERSIONS, default: Fluent::PluginHelper::Socket::TLS_DEFAULT_VERSION
85
+ desc 'The cipher configuration of TLS transport.'
86
+ config_param :tls_ciphers, :string, default: Fluent::PluginHelper::Socket::CIPHERS_DEFAULT
87
+ desc 'Skip all verification of certificates or not.'
88
+ config_param :tls_insecure_mode, :bool, default: false
89
+ desc 'Allow self signed certificates or not.'
90
+ config_param :tls_allow_self_signed_cert, :bool, default: false
91
+ desc 'Verify hostname of servers and certificates or not in TLS transport.'
92
+ config_param :tls_verify_hostname, :bool, default: true
93
+ desc 'The additional CA certificate path for TLS.'
94
+ config_param :tls_cert_path, :array, value_type: :string, default: nil
95
+
76
96
  config_section :security, required: false, multi: false do
77
97
  desc 'The hostname'
78
98
  config_param :self_hostname, :string
@@ -83,7 +103,7 @@ module Fluent::Plugin
83
103
  config_section :server, param_name: :servers do
84
104
  desc "The IP address or host name of the server."
85
105
  config_param :host, :string
86
- desc "The name of the server. Used in log messages."
106
+ desc "The name of the server. Used for logging and certificate verification in TLS transport (when host is address)."
87
107
  config_param :name, :string, default: nil
88
108
  desc "The port number of the host."
89
109
  config_param :port, :integer, default: LISTEN_PORT
@@ -134,9 +154,29 @@ module Fluent::Plugin
134
154
  @read_interval = @read_interval_msec / 1000.0
135
155
  @recover_sample_size = @recover_wait / @heartbeat_interval
136
156
 
157
+ if @heartbeat_type == :tcp
158
+ log.warn "'heartbeat_type tcp' is deprecated. use 'transport' instead."
159
+ @heartbeat_type = :transport
160
+ end
161
+
137
162
  if @dns_round_robin
138
163
  if @heartbeat_type == :udp
139
- raise Fluent::ConfigError, "forward output heartbeat type must be 'tcp' or 'none' to use dns_round_robin option"
164
+ raise Fluent::ConfigError, "forward output heartbeat type must be 'transport' or 'none' to use dns_round_robin option"
165
+ end
166
+ end
167
+
168
+ if @transport == :tls
169
+ if @tls_cert_path && !@tls_cert_path.empty?
170
+ @tls_cert_path.each do |path|
171
+ raise Fluent::ConfigError, "specified cert path does not exist:#{path}" unless File.exist?(path)
172
+ raise Fluent::ConfigError, "specified cert path is not readable:#{path}" unless File.readable?(path)
173
+ end
174
+ end
175
+
176
+ if @tls_insecure_mode
177
+ log.warn "TLS transport is configured in insecure way"
178
+ @tls_verify_hostname = false
179
+ @tls_allow_self_signed_cert = true
140
180
  end
141
181
  end
142
182
 
@@ -148,7 +188,15 @@ module Fluent::Plugin
148
188
  if @heartbeat_type == :none
149
189
  @nodes << NoneHeartbeatNode.new(self, server, failure: failure)
150
190
  else
151
- @nodes << Node.new(self, server, failure: failure)
191
+ node = Node.new(self, server, failure: failure)
192
+ begin
193
+ node.validate_host_resolution!
194
+ rescue => e
195
+ raise unless @ignore_network_errors_at_startup
196
+ log.warn "failed to resolve node name when configured", server: (server.name || server.host), error: e
197
+ node.disable!
198
+ end
199
+ @nodes << node
152
200
  end
153
201
  end
154
202
 
@@ -167,6 +215,10 @@ module Fluent::Plugin
167
215
  raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
168
216
  end
169
217
 
218
+ def multi_workers_ready?
219
+ true
220
+ end
221
+
170
222
  def prefer_delayed_commit
171
223
  @require_ack_response
172
224
  end
@@ -204,7 +256,10 @@ module Fluent::Plugin
204
256
  end
205
257
 
206
258
  def close
207
- @usock.close if @usock
259
+ if @usock
260
+ # close socket and ignore errors: this socket will not be used anyway.
261
+ @usock.close rescue nil
262
+ end
208
263
  super
209
264
  end
210
265
 
@@ -229,7 +284,7 @@ module Fluent::Plugin
229
284
  tag = chunk.metadata.tag
230
285
  sock, node = select_a_healthy_node{|n| n.send_data(tag, chunk) }
231
286
  chunk_id_base64 = Base64.encode64(chunk.unique_id)
232
- current_time = Process.clock_gettime(PROCESS_CLOCK_ID)
287
+ current_time = Fluent::Clock.now
233
288
  info = ACKWaitingSockInfo.new(sock, chunk.unique_id, chunk_id_base64, node, current_time, @ack_response_timeout)
234
289
  @sock_ack_waiting_mutex.synchronize do
235
290
  @sock_ack_waiting << info
@@ -258,14 +313,34 @@ module Fluent::Plugin
258
313
  raise NoNodesAvailable, "no nodes are available"
259
314
  end
260
315
 
261
- def create_transfer_socket(host, port, &block)
262
- socket_create_tcp(
263
- host, port,
264
- linger_timeout: @send_timeout,
265
- send_timeout: @send_timeout,
266
- recv_timeout: @ack_response_timeout,
267
- &block
268
- )
316
+ def create_transfer_socket(host, port, hostname, &block)
317
+ case @transport
318
+ when :tls
319
+ socket_create_tls(
320
+ host, port,
321
+ version: @tls_version,
322
+ ciphers: @tls_ciphers,
323
+ insecure: @tls_insecure_mode,
324
+ verify_fqdn: @tls_verify_hostname,
325
+ fqdn: hostname,
326
+ allow_self_signed_cert: @tls_allow_self_signed_cert,
327
+ cert_paths: @tls_cert_path,
328
+ linger_timeout: @send_timeout,
329
+ send_timeout: @send_timeout,
330
+ recv_timeout: @ack_response_timeout,
331
+ &block
332
+ )
333
+ when :tcp
334
+ socket_create_tcp(
335
+ host, port,
336
+ linger_timeout: @send_timeout,
337
+ send_timeout: @send_timeout,
338
+ recv_timeout: @ack_response_timeout,
339
+ &block
340
+ )
341
+ else
342
+ raise "BUG: unknown transport protocol #{@transport}"
343
+ end
269
344
  end
270
345
 
271
346
  # MessagePack FixArray length is 3
@@ -390,7 +465,7 @@ module Fluent::Plugin
390
465
  unpacker = Fluent::Engine.msgpack_unpacker
391
466
 
392
467
  while thread_current_running?
393
- now = Process.clock_gettime(PROCESS_CLOCK_ID)
468
+ now = Fluent::Clock.now
394
469
  sockets = []
395
470
  begin
396
471
  @sock_ack_waiting_mutex.synchronize do
@@ -441,6 +516,14 @@ module Fluent::Plugin
441
516
  @available = true
442
517
  @state = nil
443
518
 
519
+ # @hostname is used for certificate verification & TLS SNI
520
+ host_is_hostname = !(IPAddr.new(@host) rescue false)
521
+ @hostname = case
522
+ when host_is_hostname then @host
523
+ when @name then @name
524
+ else nil
525
+ end
526
+
444
527
  @usock = nil
445
528
 
446
529
  @username = server.username
@@ -453,7 +536,7 @@ module Fluent::Plugin
453
536
 
454
537
  @resolved_host = nil
455
538
  @resolved_time = 0
456
- resolved_host # check dns
539
+ @resolved_once = false
457
540
  end
458
541
 
459
542
  attr_accessor :usock
@@ -462,6 +545,10 @@ module Fluent::Plugin
462
545
  attr_reader :sockaddr # used by on_heartbeat
463
546
  attr_reader :failure, :available # for test
464
547
 
548
+ def validate_host_resolution!
549
+ resolved_host
550
+ end
551
+
465
552
  def available?
466
553
  @available
467
554
  end
@@ -517,20 +604,26 @@ module Fluent::Plugin
517
604
  option = { 'size' => chunk.size, 'compressed' => @compress }
518
605
  option['chunk'] = Base64.encode64(chunk.unique_id) if @sender.require_ack_response
519
606
 
520
- # out_forward always uses Raw32 type for content.
521
- # Raw16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
607
+ # https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#packedforward-mode
608
+ # out_forward always uses str32 type for entries.
609
+ # str16 can store only 64kbytes, and it should be much smaller than buffer chunk size.
610
+
611
+ tag = tag.dup.force_encoding(Encoding::UTF_8)
522
612
 
523
- sock.write @sender.forward_header # beginArray(3)
524
- sock.write tag.to_msgpack # 1. writeRaw(tag)
613
+ sock.write @sender.forward_header # array, size=3
614
+ sock.write tag.to_msgpack # 1. tag: String (str)
525
615
  chunk.open(compressed: @compress) do |chunk_io|
526
- sock.write [0xdb, chunk_io.size].pack('CN') # 2. beginRaw(size) raw32
527
- IO.copy_stream(chunk_io, sock) # writeRawBody(packed_es)
616
+ entries = [0xdb, chunk_io.size].pack('CN')
617
+ sock.write entries.force_encoding(Encoding::UTF_8) # 2. entries: String (str32)
618
+ IO.copy_stream(chunk_io, sock) # writeRawBody(packed_es)
528
619
  end
529
- sock.write option.to_msgpack # 3. writeOption(option)
620
+ sock.write option.to_msgpack # 3. option: Hash(map)
621
+
622
+ # TODO: use bin32 for non-utf8 content(entries) when old msgpack-ruby (0.5.x or earlier) not supported
530
623
  end
531
624
 
532
625
  def send_data(tag, chunk)
533
- sock = @sender.create_transfer_socket(resolved_host, port)
626
+ sock = @sender.create_transfer_socket(resolved_host, port, @hostname)
534
627
  begin
535
628
  send_data_actual(sock, tag, chunk)
536
629
  rescue
@@ -550,9 +643,20 @@ module Fluent::Plugin
550
643
 
551
644
  # FORWARD_TCP_HEARTBEAT_DATA = FORWARD_HEADER + ''.to_msgpack + [].to_msgpack
552
645
  def send_heartbeat
646
+ begin
647
+ dest_addr = resolved_host
648
+ @resolved_once = true
649
+ rescue ::SocketError => e
650
+ if !@resolved_once && @sender.ignore_network_errors_at_startup
651
+ @log.warn "failed to resolve node name in heartbeating", server: @name || @host, error: e
652
+ return
653
+ end
654
+ raise
655
+ end
656
+
553
657
  case @sender.heartbeat_type
554
- when :tcp
555
- @sender.create_transfer_socket(resolved_host, port) do |sock|
658
+ when :transport
659
+ @sender.create_transfer_socket(dest_addr, port, @hostname) do |sock|
556
660
  ## don't send any data to not cause a compatibility problem
557
661
  # sock.write FORWARD_TCP_HEARTBEAT_DATA
558
662