fluentd 1.11.5-x86-mingw32 → 1.12.4-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.deepsource.toml +13 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
  4. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  5. data/.github/workflows/linux-test.yaml +36 -0
  6. data/.github/workflows/macos-test.yaml +30 -0
  7. data/.github/workflows/stale-actions.yml +22 -0
  8. data/.github/workflows/windows-test.yaml +35 -0
  9. data/.gitlab-ci.yml +41 -19
  10. data/CHANGELOG.md +158 -0
  11. data/MAINTAINERS.md +5 -2
  12. data/README.md +7 -4
  13. data/bin/fluent-cap-ctl +7 -0
  14. data/bin/fluent-ctl +7 -0
  15. data/fluentd.gemspec +6 -4
  16. data/lib/fluent/capability.rb +87 -0
  17. data/lib/fluent/command/bundler_injection.rb +1 -1
  18. data/lib/fluent/command/ca_generate.rb +6 -3
  19. data/lib/fluent/command/cap_ctl.rb +174 -0
  20. data/lib/fluent/command/cat.rb +0 -1
  21. data/lib/fluent/command/ctl.rb +177 -0
  22. data/lib/fluent/command/fluentd.rb +4 -0
  23. data/lib/fluent/command/plugin_config_formatter.rb +18 -2
  24. data/lib/fluent/command/plugin_generator.rb +31 -1
  25. data/lib/fluent/compat/parser.rb +2 -2
  26. data/lib/fluent/config/section.rb +2 -2
  27. data/lib/fluent/config/types.rb +2 -2
  28. data/lib/fluent/env.rb +4 -0
  29. data/lib/fluent/event.rb +3 -13
  30. data/lib/fluent/load.rb +0 -1
  31. data/lib/fluent/plugin.rb +5 -0
  32. data/lib/fluent/plugin/buffer.rb +2 -21
  33. data/lib/fluent/plugin/file_wrapper.rb +39 -3
  34. data/lib/fluent/plugin/formatter.rb +2 -2
  35. data/lib/fluent/plugin/formatter_csv.rb +1 -1
  36. data/lib/fluent/plugin/formatter_hash.rb +1 -1
  37. data/lib/fluent/plugin/formatter_ltsv.rb +5 -5
  38. data/lib/fluent/plugin/formatter_out_file.rb +3 -3
  39. data/lib/fluent/plugin/formatter_single_value.rb +2 -2
  40. data/lib/fluent/plugin/formatter_tsv.rb +2 -2
  41. data/lib/fluent/plugin/in_http.rb +24 -3
  42. data/lib/fluent/plugin/in_monitor_agent.rb +1 -1
  43. data/lib/fluent/plugin/in_tail.rb +129 -41
  44. data/lib/fluent/plugin/in_tail/position_file.rb +53 -14
  45. data/lib/fluent/plugin/in_tcp.rb +1 -0
  46. data/lib/fluent/plugin/out_copy.rb +18 -5
  47. data/lib/fluent/plugin/out_exec_filter.rb +3 -3
  48. data/lib/fluent/plugin/out_forward.rb +61 -28
  49. data/lib/fluent/plugin/out_http.rb +9 -2
  50. data/lib/fluent/plugin/output.rb +18 -10
  51. data/lib/fluent/plugin/parser_csv.rb +2 -2
  52. data/lib/fluent/plugin/parser_syslog.rb +2 -2
  53. data/lib/fluent/plugin/storage_local.rb +4 -4
  54. data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
  55. data/lib/fluent/plugin_helper/inject.rb +4 -2
  56. data/lib/fluent/plugin_helper/retry_state.rb +4 -0
  57. data/lib/fluent/plugin_helper/server.rb +4 -2
  58. data/lib/fluent/plugin_helper/socket_option.rb +2 -2
  59. data/lib/fluent/supervisor.rb +153 -47
  60. data/lib/fluent/system_config.rb +2 -1
  61. data/lib/fluent/time.rb +58 -1
  62. data/lib/fluent/version.rb +1 -1
  63. data/lib/fluent/winsvc.rb +22 -4
  64. data/templates/new_gem/fluent-plugin.gemspec.erb +3 -3
  65. data/templates/plugin_config_formatter/param.md-table.erb +10 -0
  66. data/test/command/test_cap_ctl.rb +100 -0
  67. data/test/command/test_ctl.rb +57 -0
  68. data/test/command/test_fluentd.rb +38 -0
  69. data/test/command/test_plugin_config_formatter.rb +124 -2
  70. data/test/config/test_configurable.rb +1 -1
  71. data/test/plugin/in_tail/test_position_file.rb +100 -26
  72. data/test/plugin/test_file_wrapper.rb +105 -0
  73. data/test/plugin/test_in_exec.rb +1 -1
  74. data/test/plugin/test_in_http.rb +25 -0
  75. data/test/plugin/test_in_tail.rb +503 -42
  76. data/test/plugin/test_out_copy.rb +87 -0
  77. data/test/plugin/test_out_forward.rb +94 -6
  78. data/test/plugin/test_out_http.rb +20 -1
  79. data/test/plugin/test_output.rb +15 -3
  80. data/test/plugin/test_output_as_buffered_backup.rb +2 -0
  81. data/test/plugin/test_parser_csv.rb +14 -0
  82. data/test/plugin/test_parser_syslog.rb +16 -2
  83. data/test/plugin/test_sd_file.rb +1 -1
  84. data/test/plugin_helper/service_discovery/test_manager.rb +1 -1
  85. data/test/plugin_helper/test_child_process.rb +5 -2
  86. data/test/plugin_helper/test_http_server_helper.rb +4 -2
  87. data/test/plugin_helper/test_inject.rb +29 -0
  88. data/test/plugin_helper/test_server.rb +26 -7
  89. data/test/test_capability.rb +74 -0
  90. data/test/test_event.rb +16 -0
  91. data/test/test_formatter.rb +30 -0
  92. data/test/test_output.rb +2 -2
  93. data/test/test_supervisor.rb +133 -10
  94. data/test/test_time_parser.rb +109 -0
  95. metadata +85 -31
  96. data/.travis.yml +0 -57
  97. data/appveyor.yml +0 -28
@@ -98,6 +98,7 @@ module Fluent::Plugin
98
98
  def start
99
99
  super
100
100
 
101
+ log.info "listening tcp socket", bind: @bind, port: @port
101
102
  del_size = @delimiter.length
102
103
  if @_extract_enabled && @_extract_tag_key
103
104
  server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
@@ -27,20 +27,28 @@ module Fluent::Plugin
27
27
  desc 'Pass different record to each `store` plugin by specified method'
28
28
  config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
29
29
 
30
- attr_reader :ignore_errors
30
+ attr_reader :ignore_errors, :ignore_if_prev_successes
31
31
 
32
32
  def initialize
33
33
  super
34
34
  @ignore_errors = []
35
+ @ignore_if_prev_successes = []
35
36
  end
36
37
 
37
38
  def configure(conf)
38
39
  super
39
40
 
40
41
  @copy_proc = gen_copy_proc
41
- @stores.each { |store|
42
- @ignore_errors << (store.arg == 'ignore_error')
42
+ @stores.each_with_index { |store, i|
43
+ if i == 0 && store.arg.include?('ignore_if_prev_success')
44
+ raise Fluent::ConfigError, "ignore_if_prev_success must specify 2nd or later <store> directives"
45
+ end
46
+ @ignore_errors << (store.arg.include?('ignore_error'))
47
+ @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
43
48
  }
49
+ if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
50
+ log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
51
+ end
44
52
  end
45
53
 
46
54
  def multi_workers_ready?
@@ -55,10 +63,15 @@ module Fluent::Plugin
55
63
  }
56
64
  es = m
57
65
  end
58
-
66
+ success = Array.new(outputs.size)
59
67
  outputs.each_with_index do |output, i|
60
68
  begin
61
- output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
69
+ if i > 0 && success[i - 1] && @ignore_if_prev_successes[i]
70
+ log.debug "ignore copy because prev_success in #{output.plugin_id}", index: i
71
+ else
72
+ output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
73
+ success[i] = true
74
+ end
62
75
  rescue => e
63
76
  if @ignore_errors[i]
64
77
  log.error "ignore emit error in #{output.plugin_id}", error: e
@@ -159,9 +159,9 @@ module Fluent::Plugin
159
159
  @added_prefix_string = @add_prefix + '.'
160
160
  end
161
161
 
162
- @respawns = if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'
162
+ @respawns = if @child_respawn.nil? || (@child_respawn == 'none') || (@child_respawn == '0')
163
163
  0
164
- elsif @child_respawn == 'inf' or @child_respawn == '-1'
164
+ elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
165
165
  -1
166
166
  elsif @child_respawn =~ /^\d+$/
167
167
  @child_respawn.to_i
@@ -251,7 +251,7 @@ module Fluent::Plugin
251
251
 
252
252
  def tag_remove_prefix(tag)
253
253
  if @remove_prefix
254
- if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @removed_prefix_string
254
+ if ((tag[0, @removed_length] == @removed_prefix_string) && (tag.length > @removed_length)) || (tag == @removed_prefix_string)
255
255
  tag = tag[@removed_length..-1] || ''
256
256
  end
257
257
  end
@@ -166,6 +166,7 @@ module Fluent::Plugin
166
166
 
167
167
  @usock = nil
168
168
  @keep_alive_watcher_interval = 5 # TODO
169
+ @suspend_flush = false
169
170
  end
170
171
 
171
172
  def configure(conf)
@@ -291,6 +292,15 @@ module Fluent::Plugin
291
292
  @require_ack_response
292
293
  end
293
294
 
295
+ def overwrite_delayed_commit_timeout
296
+ # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
297
+ # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
298
+ if @delayed_commit_timeout != @ack_response_timeout
299
+ log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
300
+ @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
301
+ end
302
+ end
303
+
294
304
  def start
295
305
  super
296
306
 
@@ -303,13 +313,7 @@ module Fluent::Plugin
303
313
  end
304
314
 
305
315
  if @require_ack_response
306
- # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
307
- # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
308
- if @delayed_commit_timeout != @ack_response_timeout
309
- log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
310
- @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
311
- end
312
-
316
+ overwrite_delayed_commit_timeout
313
317
  thread_create(:out_forward_receiving_ack, &method(:ack_reader))
314
318
  end
315
319
 
@@ -346,6 +350,26 @@ module Fluent::Plugin
346
350
  end
347
351
  end
348
352
 
353
+ def before_shutdown
354
+ super
355
+ @suspend_flush = true
356
+ end
357
+
358
+ def after_shutdown
359
+ last_ack if @require_ack_response
360
+ super
361
+ end
362
+
363
+ def try_flush
364
+ return if @require_ack_response && @suspend_flush
365
+ super
366
+ end
367
+
368
+ def last_ack
369
+ overwrite_delayed_commit_timeout
370
+ ack_check(ack_select_interval)
371
+ end
372
+
349
373
  def write(chunk)
350
374
  return if chunk.empty?
351
375
  tag = chunk.metadata.tag
@@ -361,6 +385,7 @@ module Fluent::Plugin
361
385
  end
362
386
  tag = chunk.metadata.tag
363
387
  discovery_manager.select_service { |node| node.send_data(tag, chunk) }
388
+ last_ack if @require_ack_response && @suspend_flush
364
389
  end
365
390
 
366
391
  def create_transfer_socket(host, port, hostname, &block)
@@ -481,31 +506,39 @@ module Fluent::Plugin
481
506
  @connection_manager.purge_obsolete_socks
482
507
  end
483
508
 
509
+ def ack_select_interval
510
+ if @delayed_commit_timeout > 3
511
+ 1
512
+ else
513
+ @delayed_commit_timeout / 3.0
514
+ end
515
+ end
516
+
484
517
  def ack_reader
485
- select_interval = if @delayed_commit_timeout > 3
486
- 1
487
- else
488
- @delayed_commit_timeout / 3.0
489
- end
518
+ select_interval = ack_select_interval
490
519
 
491
520
  while thread_current_running?
492
- @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
493
- @connection_manager.close(sock)
494
-
495
- case result
496
- when AckHandler::Result::SUCCESS
497
- commit_write(chunk_id)
498
- when AckHandler::Result::FAILED
499
- node.disable!
500
- rollback_write(chunk_id, update_retry: false)
501
- when AckHandler::Result::CHUNKID_UNMATCHED
502
- rollback_write(chunk_id, update_retry: false)
503
- else
504
- log.warn("BUG: invalid status #{result} #{chunk_id}")
521
+ ack_check(select_interval)
522
+ end
523
+ end
505
524
 
506
- if chunk_id
507
- rollback_write(chunk_id, update_retry: false)
508
- end
525
+ def ack_check(select_interval)
526
+ @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
527
+ @connection_manager.close(sock)
528
+
529
+ case result
530
+ when AckHandler::Result::SUCCESS
531
+ commit_write(chunk_id)
532
+ when AckHandler::Result::FAILED
533
+ node.disable!
534
+ rollback_write(chunk_id, update_retry: false)
535
+ when AckHandler::Result::CHUNKID_UNMATCHED
536
+ rollback_write(chunk_id, update_retry: false)
537
+ else
538
+ log.warn("BUG: invalid status #{result} #{chunk_id}")
539
+
540
+ if chunk_id
541
+ rollback_write(chunk_id, update_retry: false)
509
542
  end
510
543
  end
511
544
  end
@@ -51,6 +51,8 @@ module Fluent::Plugin
51
51
  config_param :json_array, :bool, default: false
52
52
  desc 'Additional headers for HTTP request'
53
53
  config_param :headers, :hash, default: nil
54
+ desc 'Additional placeholder based headers for HTTP request'
55
+ config_param :headers_from_placeholders, :hash, default: nil
54
56
 
55
57
  desc 'The connection open timeout in seconds'
56
58
  config_param :open_timeout, :integer, default: nil
@@ -213,12 +215,17 @@ module Fluent::Plugin
213
215
  URI.parse(endpoint)
214
216
  end
215
217
 
216
- def set_headers(req)
218
+ def set_headers(req, chunk)
217
219
  if @headers
218
220
  @headers.each do |k, v|
219
221
  req[k] = v
220
222
  end
221
223
  end
224
+ if @headers_from_placeholders
225
+ @headers_from_placeholders.each do |k, v|
226
+ req[k] = extract_placeholders(v, chunk)
227
+ end
228
+ end
222
229
  req['Content-Type'] = @content_type
223
230
  end
224
231
 
@@ -232,7 +239,7 @@ module Fluent::Plugin
232
239
  if @auth
233
240
  req.basic_auth(@auth.username, @auth.password)
234
241
  end
235
- set_headers(req)
242
+ set_headers(req, chunk)
236
243
  req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
237
244
  req
238
245
  end
@@ -754,7 +754,17 @@ module Fluent
754
754
  log.warn "tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}"
755
755
  end
756
756
  end
757
- # ${a_chunk_key}, ...
757
+
758
+ # First we replace ${chunk_id} with chunk.unique_id (hexlified).
759
+ rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
760
+ if chunk_passed
761
+ dump_unique_id_hex(chunk.unique_id)
762
+ else
763
+ log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
764
+ end
765
+ }
766
+
767
+ # Then, replace other ${chunk_key}s.
758
768
  if !@chunk_keys.empty? && metadata.variables
759
769
  hash = {'${tag}' => '${tag}'} # not to erase this wrongly
760
770
  @chunk_keys.each do |key|
@@ -769,14 +779,6 @@ module Fluent
769
779
  end
770
780
  end
771
781
 
772
- rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
773
- if chunk_passed
774
- dump_unique_id_hex(chunk.unique_id)
775
- else
776
- log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
777
- end
778
- }
779
-
780
782
  if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
781
783
  log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
782
784
  end
@@ -1252,7 +1254,13 @@ module Fluent
1252
1254
  log.debug "buffer queue cleared"
1253
1255
  @retry = nil
1254
1256
  else
1255
- @retry.step
1257
+ # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
1258
+ # @retry.step is called almost as many times as the number of flush threads in a short time.
1259
+ if Time.now >= @retry.next_time
1260
+ @retry.step
1261
+ else
1262
+ @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
1263
+ end
1256
1264
  if error
1257
1265
  if using_secondary
1258
1266
  msg = "failed to flush the buffer with secondary output."
@@ -28,13 +28,13 @@ module Fluent
28
28
  desc 'The delimiter character (or string) of CSV values'
29
29
  config_param :delimiter, :string, default: ','
30
30
  desc 'The parser type used to parse CSV line'
31
- config_param :parser_type, :enum, list: [:normal, :fast], default: :normal
31
+ config_param :parser_engine, :enum, list: [:normal, :fast], default: :normal, alias: :parser_type
32
32
 
33
33
  def configure(conf)
34
34
  super
35
35
 
36
36
 
37
- if @parser_type == :fast
37
+ if @parser_engine == :fast
38
38
  @quote_char = '"'
39
39
  @escape_pattern = Regexp.compile(@quote_char * 2)
40
40
 
@@ -56,7 +56,7 @@ module Fluent
56
56
  desc 'Specify time format for event time for rfc5424 protocol'
57
57
  config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
58
58
  desc 'The parser type used to parse syslog message'
59
- config_param :parser_type, :enum, list: [:regexp, :string], default: :regexp
59
+ config_param :parser_engine, :enum, list: [:regexp, :string], default: :regexp, alias: :parser_type
60
60
  desc 'support colonless ident in string parser'
61
61
  config_param :support_colonless_ident, :bool, default: true
62
62
 
@@ -79,7 +79,7 @@ module Fluent
79
79
  def configure(conf)
80
80
  super
81
81
 
82
- @regexp_parser = @parser_type == :regexp
82
+ @regexp_parser = @parser_engine == :regexp
83
83
  @regexp = case @message_format
84
84
  when :rfc3164
85
85
  if @regexp_parser
@@ -87,7 +87,7 @@ module Fluent
87
87
  if File.exist?(@path)
88
88
  raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
89
89
  begin
90
- data = open(@path, 'r:utf-8') { |io| io.read }
90
+ data = File.open(@path, 'r:utf-8') { |io| io.read }
91
91
  if data.empty?
92
92
  log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
93
93
  return
@@ -115,7 +115,7 @@ module Fluent
115
115
  return if @on_memory
116
116
  return unless File.exist?(@path)
117
117
  begin
118
- json_string = open(@path, 'r:utf-8'){ |io| io.read }
118
+ json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
119
119
  json = Yajl::Parser.parse(json_string)
120
120
  unless json.is_a?(Hash)
121
121
  log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
@@ -130,10 +130,10 @@ module Fluent
130
130
 
131
131
  def save
132
132
  return if @on_memory
133
- tmp_path = @path + '.tmp'
133
+ tmp_path = @path + '.tmp.' + Fluent::UniqueId.hex(Fluent::UniqueId.generate)
134
134
  begin
135
135
  json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
136
- open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
136
+ File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
137
137
  File.rename(tmp_path, @path)
138
138
  rescue => e
139
139
  log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e
@@ -81,7 +81,7 @@ module Fluent
81
81
 
82
82
  def build_handler
83
83
  @methods.group_by(&:first).each do |(path, rest)|
84
- klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
84
+ klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
85
85
  @server.mount(path, klass)
86
86
  end
87
87
  end
@@ -76,7 +76,7 @@ module Fluent
76
76
  config_param :time_key, :string, default: nil
77
77
 
78
78
  # To avoid defining :time_type twice
79
- config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :string], default: :float
79
+ config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :unixtime_micros, :unixtime_nanos, :string], default: :float
80
80
 
81
81
  Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
82
82
  config_param(name, type, **opts)
@@ -132,7 +132,9 @@ module Fluent
132
132
  if @_inject_time_key
133
133
  @_inject_time_formatter = case @inject_config.time_type
134
134
  when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
135
- when :unixtime_millis then ->(time) { time.to_f.floor(3) * 1000 }
135
+ when :unixtime_millis then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000 + time.nsec / 1_000_000 : (time * 1_000).floor }
136
+ when :unixtime_micros then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000 + time.nsec / 1_000 : (time * 1_000_000).floor }
137
+ when :unixtime_nanos then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000_000 + time.nsec : (time * 1_000_000_000).floor }
136
138
  when :unixtime then ->(time){ time.to_i }
137
139
  else
138
140
  localtime = @inject_config.localtime && !@inject_config.utc
@@ -127,6 +127,10 @@ module Fluent
127
127
  nil
128
128
  end
129
129
 
130
+ def recalc_next_time
131
+ @next_time = calc_next_time
132
+ end
133
+
130
134
  def limit?
131
135
  if @forever
132
136
  false
@@ -709,11 +709,13 @@ module Fluent
709
709
  return true
710
710
  end
711
711
  rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
712
- @log.trace "unexpected error before accepting TLS connection", error: e
712
+ @log.trace "unexpected error before accepting TLS connection",
713
+ host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
713
714
  close rescue nil
714
715
  rescue OpenSSL::SSL::SSLError => e
715
716
  # Use same log level as on_readable
716
- @log.warn "unexpected error before accepting TLS connection by OpenSSL", error: e
717
+ @log.warn "unexpected error before accepting TLS connection by OpenSSL",
718
+ host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
717
719
  close rescue nil
718
720
  end
719
721
 
@@ -38,8 +38,8 @@ module Fluent
38
38
  end
39
39
  end
40
40
  if send_keepalive_packet
41
- if protocol != :tcp
42
- raise ArgumentError, "BUG: send_keepalive_packet is available for tcp"
41
+ if protocol != :tcp && protocol != :tls
42
+ raise ArgumentError, "BUG: send_keepalive_packet is available for tcp/tls"
43
43
  end
44
44
  end
45
45
  end
@@ -45,6 +45,7 @@ module Fluent
45
45
  module ServerModule
46
46
  def before_run
47
47
  @fluentd_conf = config[:fluentd_conf]
48
+ @rpc_endpoint = nil
48
49
  @rpc_server = nil
49
50
  @counter = nil
50
51
 
@@ -53,23 +54,28 @@ module Fluent
53
54
  @enable_get_dump = config[:enable_get_dump]
54
55
  run_rpc_server
55
56
  end
56
- install_supervisor_signal_handlers
57
57
 
58
- if config[:signame]
59
- @signame = config[:signame]
58
+ if Fluent.windows?
60
59
  install_windows_event_handler
60
+ else
61
+ install_supervisor_signal_handlers
61
62
  end
62
63
 
63
64
  if counter = config[:counter_server]
64
65
  run_counter_server(counter)
65
66
  end
66
67
 
67
- socket_manager_path = ServerEngine::SocketManager::Server.generate_path
68
- ServerEngine::SocketManager::Server.open(socket_manager_path)
69
- ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
68
+ if config[:disable_shared_socket]
69
+ $log.info "shared socket for multiple workers is disabled"
70
+ else
71
+ socket_manager_path = ServerEngine::SocketManager::Server.generate_path
72
+ ServerEngine::SocketManager::Server.open(socket_manager_path)
73
+ ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
74
+ end
70
75
  end
71
76
 
72
77
  def after_run
78
+ stop_windows_event_thread if Fluent.windows?
73
79
  stop_rpc_server if @rpc_endpoint
74
80
  stop_counter_server if @counter
75
81
  Fluent::Supervisor.cleanup_resources
@@ -92,7 +98,8 @@ module Fluent
92
98
  @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
93
99
  $log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
94
100
  if Fluent.windows?
95
- $log.warn "operation 'flushBuffersAndKillWorkers' is not supported on Windows now."
101
+ supervisor_sigusr1_handler
102
+ stop(true)
96
103
  else
97
104
  Process.kill :USR1, $$
98
105
  Process.kill :TERM, $$
@@ -101,7 +108,9 @@ module Fluent
101
108
  }
102
109
  @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
103
110
  $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
104
- unless Fluent.windows?
111
+ if Fluent.windows?
112
+ supervisor_sigusr1_handler
113
+ else
105
114
  Process.kill :USR1, $$
106
115
  end
107
116
  nil
@@ -125,7 +134,9 @@ module Fluent
125
134
 
126
135
  @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
127
136
  $log.debug "fluentd RPC got /api/config.gracefulReload request"
128
- unless Fluent.windows?
137
+ if Fluent.windows?
138
+ supervisor_sigusr2_handler
139
+ else
129
140
  Process.kill :USR2, $$
130
141
  end
131
142
 
@@ -159,37 +170,101 @@ module Fluent
159
170
  end
160
171
 
161
172
  def install_supervisor_signal_handlers
173
+ return if Fluent.windows?
174
+
162
175
  trap :HUP do
163
176
  $log.debug "fluentd supervisor process get SIGHUP"
164
177
  supervisor_sighup_handler
165
- end unless Fluent.windows?
178
+ end
166
179
 
167
180
  trap :USR1 do
168
181
  $log.debug "fluentd supervisor process get SIGUSR1"
169
182
  supervisor_sigusr1_handler
170
- end unless Fluent.windows?
183
+ end
171
184
 
172
185
  trap :USR2 do
173
186
  $log.debug 'fluentd supervisor process got SIGUSR2'
174
187
  supervisor_sigusr2_handler
175
- end unless Fluent.windows?
188
+ end
189
+ end
190
+
191
+ if Fluent.windows?
192
+ # Override some methods of ServerEngine::MultiSpawnWorker
193
+ # Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2
194
+ # handlers (see install_supervisor_signal_handlers), they should be
195
+ # disabled also on Windows, just send commands to workers instead.
196
+ def restart(graceful)
197
+ @monitors.each do |m|
198
+ m.send_command(graceful ? "GRACEFUL_RESTART\n" : "IMMEDIATE_RESTART\n")
199
+ end
200
+ end
201
+
202
+ def reload
203
+ @monitors.each do |m|
204
+ m.send_command("RELOAD\n")
205
+ end
206
+ end
176
207
  end
177
208
 
178
209
  def install_windows_event_handler
210
+ return unless Fluent.windows?
211
+
212
+ @pid_signame = "fluentd_#{$$}"
213
+ @signame = config[:signame]
214
+
179
215
  Thread.new do
180
- ev = Win32::Event.new(@signame)
216
+ ipc = Win32::Ipc.new(nil)
217
+ events = [
218
+ Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
219
+ Win32::Event.new("#{@pid_signame}"),
220
+ Win32::Event.new("#{@pid_signame}_HUP"),
221
+ Win32::Event.new("#{@pid_signame}_USR1"),
222
+ Win32::Event.new("#{@pid_signame}_USR2"),
223
+ ]
224
+ if @signame
225
+ signame_events = [
226
+ Win32::Event.new("#{@signame}"),
227
+ Win32::Event.new("#{@signame}_HUP"),
228
+ Win32::Event.new("#{@signame}_USR1"),
229
+ Win32::Event.new("#{@signame}_USR2"),
230
+ ]
231
+ events.concat(signame_events)
232
+ end
181
233
  begin
182
- ev.reset
183
- until WaitForSingleObject(ev.handle, 0) == WAIT_OBJECT_0
184
- sleep 1
234
+ loop do
235
+ idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
236
+ if idx > 0 && idx <= events.length
237
+ $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
238
+ else
239
+ $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
240
+ end
241
+ case idx
242
+ when 2, 6
243
+ stop(true)
244
+ when 3, 7
245
+ supervisor_sighup_handler
246
+ when 4, 8
247
+ supervisor_sigusr1_handler
248
+ when 5, 9
249
+ supervisor_sigusr2_handler
250
+ when 1
251
+ break
252
+ end
185
253
  end
186
- stop(true)
187
254
  ensure
188
- ev.close
255
+ events.each { |event| event.close }
189
256
  end
190
257
  end
191
258
  end
192
259
 
260
+ def stop_windows_event_thread
261
+ if Fluent.windows?
262
+ ev = Win32::Event.open("#{@pid_signame}_STOP_EVENT_THREAD")
263
+ ev.set
264
+ ev.close
265
+ end
266
+ end
267
+
193
268
  def supervisor_sighup_handler
194
269
  kill_worker
195
270
  end
@@ -264,9 +339,25 @@ module Fluent
264
339
  def send_signal_to_workers(signal)
265
340
  return unless config[:worker_pid]
266
341
 
267
- config[:worker_pid].each_value do |pid|
268
- # don't rescue Errno::ESRCH here (invalid status)
269
- Process.kill(signal, pid)
342
+ if Fluent.windows?
343
+ send_command_to_workers(signal)
344
+ else
345
+ config[:worker_pid].each_value do |pid|
346
+ # don't rescue Errno::ESRCH here (invalid status)
347
+ Process.kill(signal, pid)
348
+ end
349
+ end
350
+ end
351
+
352
+ def send_command_to_workers(signal)
353
+ # Use SeverEngine's CommandSender on Windows
354
+ case signal
355
+ when :HUP
356
+ restart(false)
357
+ when :USR1
358
+ restart(true)
359
+ when :USR2
360
+ reload
270
361
  end
271
362
  end
272
363
  end
@@ -294,7 +385,7 @@ module Fluent
294
385
  config_mtime = File.mtime(path)
295
386
 
296
387
  # reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
297
- if Time.now - Time.at(pre_loadtime) < 5 and config_mtime == pre_config_mtime
388
+ if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
298
389
  return params['pre_conf']
299
390
  end
300
391
 
@@ -366,6 +457,7 @@ module Fluent
366
457
  config_path: path,
367
458
  main_cmd: params['main_cmd'],
368
459
  signame: params['signame'],
460
+ disable_shared_socket: params['disable_shared_socket']
369
461
  }
370
462
  if daemonize
371
463
  se_config[:pid_path] = pid_path
@@ -481,7 +573,8 @@ module Fluent
481
573
  supervise: true,
482
574
  standalone_worker: false,
483
575
  signame: nil,
484
- conf_encoding: 'utf-8'
576
+ conf_encoding: 'utf-8',
577
+ disable_shared_socket: nil
485
578
  }
486
579
  end
487
580
 
@@ -709,6 +802,7 @@ module Fluent
709
802
  'counter_server' => @system_config.counter_server,
710
803
  'log_format' => @system_config.log.format,
711
804
  'log_time_format' => @system_config.log.time_format,
805
+ 'disable_shared_socket' => @system_config.disable_shared_socket
712
806
  }
713
807
 
714
808
  se = ServerEngine.create(ServerModule, WorkerModule){
@@ -745,33 +839,45 @@ module Fluent
745
839
  end
746
840
  end
747
841
 
748
- trap :USR1 do
749
- flush_buffer
750
- end unless Fluent.windows?
842
+ if Fluent.windows?
843
+ install_main_process_command_handlers
844
+ else
845
+ trap :USR1 do
846
+ flush_buffer
847
+ end
751
848
 
752
- trap :USR2 do
753
- reload_config
754
- end unless Fluent.windows?
849
+ trap :USR2 do
850
+ reload_config
851
+ end
852
+ end
853
+ end
755
854
 
756
- if Fluent.windows?
757
- command_pipe = STDIN.dup
758
- STDIN.reopen(File::NULL, "rb")
759
- command_pipe.binmode
760
- command_pipe.sync = true
855
+ def install_main_process_command_handlers
856
+ command_pipe = $stdin.dup
857
+ $stdin.reopen(File::NULL, "rb")
858
+ command_pipe.binmode
859
+ command_pipe.sync = true
761
860
 
762
- Thread.new do
763
- loop do
764
- cmd = command_pipe.gets.chomp
765
- case cmd
766
- when "GRACEFUL_STOP", "IMMEDIATE_STOP"
767
- $log.debug "fluentd main process get #{cmd} command"
768
- @finished = true
769
- $log.debug "getting start to shutdown main process"
770
- Fluent::Engine.stop
771
- break
772
- else
773
- $log.warn "fluentd main process get unknown command [#{cmd}]"
774
- end
861
+ Thread.new do
862
+ loop do
863
+ cmd = command_pipe.gets
864
+ break unless cmd
865
+
866
+ case cmd.chomp!
867
+ when "GRACEFUL_STOP", "IMMEDIATE_STOP"
868
+ $log.debug "fluentd main process get #{cmd} command"
869
+ @finished = true
870
+ $log.debug "getting start to shutdown main process"
871
+ Fluent::Engine.stop
872
+ break
873
+ when "GRACEFUL_RESTART"
874
+ $log.debug "fluentd main process get #{cmd} command"
875
+ flush_buffer
876
+ when "RELOAD"
877
+ $log.debug "fluentd main process get #{cmd} command"
878
+ reload_config
879
+ else
880
+ $log.warn "fluentd main process get unknown command [#{cmd}]"
775
881
  end
776
882
  end
777
883
  end