fluentd 1.11.1-x86-mingw32 → 1.12.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
  3. data/.github/ISSUE_TEMPLATE/config.yml +5 -0
  4. data/.github/workflows/stale-actions.yml +22 -0
  5. data/.travis.yml +22 -2
  6. data/CHANGELOG.md +111 -0
  7. data/README.md +1 -1
  8. data/appveyor.yml +3 -0
  9. data/bin/fluent-cap-ctl +7 -0
  10. data/bin/fluent-ctl +7 -0
  11. data/example/copy_roundrobin.conf +3 -3
  12. data/example/counter.conf +1 -1
  13. data/example/filter_stdout.conf +2 -2
  14. data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
  15. data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
  16. data/example/logevents.conf +5 -5
  17. data/example/multi_filters.conf +1 -1
  18. data/example/out_exec_filter.conf +2 -2
  19. data/example/out_forward.conf +1 -1
  20. data/example/out_forward_buf_file.conf +1 -1
  21. data/example/out_forward_client.conf +5 -5
  22. data/example/out_forward_heartbeat_none.conf +1 -1
  23. data/example/out_forward_sd.conf +1 -1
  24. data/example/out_forward_shared_key.conf +2 -2
  25. data/example/out_forward_tls.conf +1 -1
  26. data/example/out_forward_users.conf +3 -3
  27. data/example/out_null.conf +4 -4
  28. data/example/secondary_file.conf +1 -1
  29. data/fluentd.gemspec +7 -6
  30. data/lib/fluent/capability.rb +87 -0
  31. data/lib/fluent/command/cap_ctl.rb +174 -0
  32. data/lib/fluent/command/ctl.rb +177 -0
  33. data/lib/fluent/command/plugin_config_formatter.rb +2 -1
  34. data/lib/fluent/env.rb +4 -0
  35. data/lib/fluent/log.rb +33 -3
  36. data/lib/fluent/match.rb +9 -0
  37. data/lib/fluent/plugin.rb +5 -0
  38. data/lib/fluent/plugin/buffer.rb +32 -42
  39. data/lib/fluent/plugin/buffer/chunk.rb +2 -1
  40. data/lib/fluent/plugin/formatter.rb +24 -0
  41. data/lib/fluent/plugin/formatter_csv.rb +1 -1
  42. data/lib/fluent/plugin/formatter_hash.rb +3 -1
  43. data/lib/fluent/plugin/formatter_json.rb +3 -1
  44. data/lib/fluent/plugin/formatter_ltsv.rb +5 -3
  45. data/lib/fluent/plugin/formatter_out_file.rb +6 -4
  46. data/lib/fluent/plugin/formatter_single_value.rb +4 -2
  47. data/lib/fluent/plugin/formatter_tsv.rb +4 -2
  48. data/lib/fluent/plugin/in_dummy.rb +2 -123
  49. data/lib/fluent/plugin/in_exec.rb +4 -2
  50. data/lib/fluent/plugin/in_http.rb +25 -4
  51. data/lib/fluent/plugin/in_sample.rb +141 -0
  52. data/lib/fluent/plugin/in_tail.rb +109 -41
  53. data/lib/fluent/plugin/in_tail/position_file.rb +39 -14
  54. data/lib/fluent/plugin/in_tcp.rb +1 -0
  55. data/lib/fluent/plugin/out_http.rb +20 -2
  56. data/lib/fluent/plugin/output.rb +15 -6
  57. data/lib/fluent/plugin/parser_json.rb +5 -2
  58. data/lib/fluent/plugin_helper/cert_option.rb +5 -8
  59. data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
  60. data/lib/fluent/plugin_helper/inject.rb +4 -1
  61. data/lib/fluent/plugin_helper/retry_state.rb +4 -0
  62. data/lib/fluent/plugin_helper/socket.rb +1 -1
  63. data/lib/fluent/supervisor.rb +151 -48
  64. data/lib/fluent/system_config.rb +2 -1
  65. data/lib/fluent/time.rb +1 -0
  66. data/lib/fluent/version.rb +1 -1
  67. data/lib/fluent/winsvc.rb +22 -4
  68. data/test/command/test_binlog_reader.rb +22 -6
  69. data/test/command/test_cap_ctl.rb +100 -0
  70. data/test/command/test_ctl.rb +57 -0
  71. data/test/command/test_plugin_config_formatter.rb +57 -2
  72. data/test/plugin/in_tail/test_position_file.rb +45 -25
  73. data/test/plugin/test_buffer.rb +4 -0
  74. data/test/plugin/test_filter_stdout.rb +6 -1
  75. data/test/plugin/test_formatter_hash.rb +6 -3
  76. data/test/plugin/test_formatter_json.rb +14 -4
  77. data/test/plugin/test_formatter_ltsv.rb +13 -5
  78. data/test/plugin/test_formatter_out_file.rb +35 -14
  79. data/test/plugin/test_formatter_single_value.rb +12 -6
  80. data/test/plugin/test_formatter_tsv.rb +12 -4
  81. data/test/plugin/test_in_exec.rb +18 -0
  82. data/test/plugin/test_in_http.rb +25 -0
  83. data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
  84. data/test/plugin/test_in_tail.rb +433 -30
  85. data/test/plugin/test_out_file.rb +23 -18
  86. data/test/plugin/test_output.rb +12 -0
  87. data/test/plugin/test_parser_syslog.rb +2 -2
  88. data/test/plugin_helper/data/cert/empty.pem +0 -0
  89. data/test/plugin_helper/test_cert_option.rb +7 -0
  90. data/test/plugin_helper/test_compat_parameters.rb +7 -2
  91. data/test/plugin_helper/test_http_server_helper.rb +5 -0
  92. data/test/plugin_helper/test_inject.rb +42 -0
  93. data/test/plugin_helper/test_server.rb +34 -0
  94. data/test/plugin_helper/test_socket.rb +8 -0
  95. data/test/test_capability.rb +74 -0
  96. data/test/test_formatter.rb +34 -10
  97. data/test/test_log.rb +44 -0
  98. data/test/test_match.rb +11 -0
  99. data/test/test_output.rb +6 -1
  100. data/test/test_static_config_analysis.rb +2 -2
  101. data/test/test_supervisor.rb +119 -1
  102. metadata +50 -18
@@ -22,34 +22,40 @@ module Fluent::Plugin
22
22
  UNWATCHED_POSITION = 0xffffffffffffffff
23
23
  POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
24
24
 
25
- def self.load(file, logger:)
26
- pf = new(file, logger: logger)
25
+ def self.load(file, follow_inodes, existing_paths, logger:)
26
+ pf = new(file, follow_inodes, existing_paths, logger: logger)
27
27
  pf.load
28
28
  pf
29
29
  end
30
30
 
31
- def initialize(file, logger: nil)
31
+ def initialize(file, follow_inodes, existing_paths, logger: nil)
32
32
  @file = file
33
33
  @logger = logger
34
34
  @file_mutex = Mutex.new
35
35
  @map = {}
36
+ @follow_inodes = follow_inodes
37
+ @existing_paths = existing_paths
36
38
  end
37
39
 
38
- def [](path)
39
- if m = @map[path]
40
+ def [](target_info)
41
+ if m = @map[@follow_inodes ? target_info.ino : target_info.path]
40
42
  return m
41
43
  end
42
44
 
43
45
  @file_mutex.synchronize {
44
46
  @file.seek(0, IO::SEEK_END)
45
- seek = @file.pos + path.bytesize + 1
46
- @file.write "#{path}\t0000000000000000\t0000000000000000\n"
47
- @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
47
+ seek = @file.pos + target_info.path.bytesize + 1
48
+ @file.write "#{target_info.path}\t0000000000000000\t0000000000000000\n"
49
+ if @follow_inodes
50
+ @map[target_info.ino] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
51
+ else
52
+ @map[target_info.path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
53
+ end
48
54
  }
49
55
  end
50
56
 
51
- def unwatch(path)
52
- if (entry = @map.delete(path))
57
+ def unwatch(target_info)
58
+ if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
53
59
  entry.update_pos(UNWATCHED_POSITION)
54
60
  end
55
61
  end
@@ -69,7 +75,11 @@ module Fluent::Plugin
69
75
  pos = m[2].to_i(16)
70
76
  ino = m[3].to_i(16)
71
77
  seek = @file.pos - line.bytesize + path.bytesize + 1
72
- map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
78
+ if @follow_inodes
79
+ map[ino] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
80
+ else
81
+ map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
82
+ end
73
83
  end
74
84
  end
75
85
 
@@ -94,8 +104,9 @@ module Fluent::Plugin
94
104
  @file.truncate(0)
95
105
  @file.write(entries.values.map(&:to_entry_fmt).join)
96
106
 
97
- entries.each do |path, val|
98
- if (m = @map[path])
107
+ # entry contains path/ino key and value.
108
+ entries.each do |key, val|
109
+ if (m = @map[key])
99
110
  m.seek = val.seek
100
111
  end
101
112
  end
@@ -139,13 +150,25 @@ module Fluent::Plugin
139
150
  @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
140
151
  end
141
152
 
142
- entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
153
+ if @follow_inodes
154
+ entries[ino] = Entry.new(path, pos, ino, file_pos + path.size + 1)
155
+ else
156
+ entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
157
+ end
143
158
  file_pos += line.size
144
159
  end
145
160
  end
146
161
 
162
+ entries = remove_deleted_files_entries(entries, @existing_paths) if @follow_inodes
147
163
  entries
148
164
  end
165
+
166
+ def remove_deleted_files_entries(existent_entries, existing_paths)
167
+ filtered_entries = existent_entries.select {|file_entry|
168
+ existing_paths.key?(file_entry)
169
+ }
170
+ filtered_entries
171
+ end
149
172
  end
150
173
 
151
174
  Entry = Struct.new(:path, :pos, :ino, :seek) do
@@ -224,5 +247,7 @@ module Fluent::Plugin
224
247
  @inode
225
248
  end
226
249
  end
250
+
251
+ TargetInfo = Struct.new(:path, :ino)
227
252
  end
228
253
  end
@@ -98,6 +98,7 @@ module Fluent::Plugin
98
98
  def start
99
99
  super
100
100
 
101
+ log.info "listening tcp socket", bind: @bind, port: @port
101
102
  del_size = @delimiter.length
102
103
  if @_extract_enabled && @_extract_tag_key
103
104
  server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
@@ -21,6 +21,16 @@ require 'fluent/tls'
21
21
  require 'fluent/plugin/output'
22
22
  require 'fluent/plugin_helper/socket'
23
23
 
24
+ # patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758.
25
+ # see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712
26
+ unless Net::HTTP::SSL_IVNAMES.include?(:@extra_chain_cert)
27
+ class Net::HTTP
28
+ SSL_IVNAMES << :@extra_chain_cert
29
+ SSL_ATTRIBUTES << :extra_chain_cert
30
+ attr_accessor :extra_chain_cert
31
+ end
32
+ end
33
+
24
34
  module Fluent::Plugin
25
35
  class HTTPOutput < Output
26
36
  Fluent::Plugin.register_output('http', self)
@@ -171,7 +181,15 @@ module Fluent::Plugin
171
181
  end
172
182
  if @tls_client_cert_path
173
183
  raise Fluent::ConfigError, "tls_client_cert_path is wrong: #{@tls_client_cert_path}" unless File.file?(@tls_client_cert_path)
174
- opt[:cert] = OpenSSL::X509::Certificate.new(File.read(@tls_client_cert_path))
184
+
185
+ bundle = File.read(@tls_client_cert_path)
186
+ bundle_certs = bundle.scan(/-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/)
187
+ opt[:cert] = OpenSSL::X509::Certificate.new(bundle_certs[0])
188
+
189
+ intermediate_certs = bundle_certs[1..-1]
190
+ if intermediate_certs
191
+ opt[:extra_chain_cert] = intermediate_certs.map { |cert| OpenSSL::X509::Certificate.new(cert) }
192
+ end
175
193
  end
176
194
  if @tls_private_key_path
177
195
  raise Fluent::ConfigError, "tls_private_key_path is wrong: #{@tls_private_key_path}" unless File.file?(@tls_private_key_path)
@@ -215,7 +233,7 @@ module Fluent::Plugin
215
233
  req.basic_auth(@auth.username, @auth.password)
216
234
  end
217
235
  set_headers(req)
218
- req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
236
+ req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
219
237
  req
220
238
  end
221
239
 
@@ -340,6 +340,7 @@ module Fluent
340
340
  buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
341
341
  @buffer = Plugin.new_buffer(buffer_type, parent: self)
342
342
  @buffer.configure(buffer_conf)
343
+ @buffer.enable_update_timekeys if @chunk_key_time
343
344
 
344
345
  @flush_at_shutdown = @buffer_config.flush_at_shutdown
345
346
  if @flush_at_shutdown.nil?
@@ -768,17 +769,19 @@ module Fluent
768
769
  end
769
770
  end
770
771
 
771
- if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
772
- log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
773
- end
774
-
775
- rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
772
+ rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
776
773
  if chunk_passed
777
774
  dump_unique_id_hex(chunk.unique_id)
778
775
  else
779
776
  log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
780
777
  end
781
778
  }
779
+
780
+ if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
781
+ log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
782
+ end
783
+
784
+ rvalue
782
785
  end
783
786
  end
784
787
 
@@ -1249,7 +1252,13 @@ module Fluent
1249
1252
  log.debug "buffer queue cleared"
1250
1253
  @retry = nil
1251
1254
  else
1252
- @retry.step
1255
+ # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
1256
+ # @retry.step is called almost as many times as the number of flush threads in a short time.
1257
+ if Time.now >= @retry.next_time
1258
+ @retry.step
1259
+ else
1260
+ @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
1261
+ end
1253
1262
  if error
1254
1263
  if using_secondary
1255
1264
  msg = "failed to flush the buffer with secondary output."
@@ -71,8 +71,11 @@ module Fluent
71
71
  end
72
72
 
73
73
  def parse(text)
74
- r = @load_proc.call(text)
75
- time, record = convert_values(parse_time(r), r)
74
+ record = @load_proc.call(text)
75
+ time = parse_time(record)
76
+ if @execute_convert_values
77
+ time, record = convert_values(time, record)
78
+ end
76
79
  yield time, record
77
80
  rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
78
81
  yield nil, nil
@@ -27,13 +27,9 @@ module Fluent
27
27
  cert, key, extra = cert_option_server_validate!(conf)
28
28
 
29
29
  ctx = OpenSSL::SSL::SSLContext.new
30
- unless insecure
31
- # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
32
- # https://bugs.ruby-lang.org/issues/9424
33
- ctx.set_params({})
34
-
35
- ctx.ciphers = ciphers
36
- end
30
+ # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
31
+ # https://bugs.ruby-lang.org/issues/9424
32
+ ctx.set_params({}) unless insecure
37
33
 
38
34
  if conf.client_cert_auth
39
35
  ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
@@ -56,6 +52,7 @@ module Fluent
56
52
  end
57
53
 
58
54
  Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
55
+ ctx.ciphers = ciphers unless insecure
59
56
 
60
57
  ctx
61
58
  end
@@ -185,7 +182,7 @@ module Fluent
185
182
  list = []
186
183
  data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
187
184
  if list.length == 0
188
- log.warn "cert_path does not contain a valid certificate"
185
+ raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
189
186
  end
190
187
  list
191
188
  end
@@ -81,7 +81,7 @@ module Fluent
81
81
 
82
82
  def build_handler
83
83
  @methods.group_by(&:first).each do |(path, rest)|
84
- klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
84
+ klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
85
85
  @server.mount(path, klass)
86
86
  end
87
87
  end
@@ -76,7 +76,7 @@ module Fluent
76
76
  config_param :time_key, :string, default: nil
77
77
 
78
78
  # To avoid defining :time_type twice
79
- config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
79
+ config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :unixtime_micros, :unixtime_nanos, :string], default: :float
80
80
 
81
81
  Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
82
82
  config_param(name, type, **opts)
@@ -132,6 +132,9 @@ module Fluent
132
132
  if @_inject_time_key
133
133
  @_inject_time_formatter = case @inject_config.time_type
134
134
  when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
135
+ when :unixtime_millis then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000 + time.nsec / 1_000_000 : (time * 1_000).floor }
136
+ when :unixtime_micros then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000 + time.nsec / 1_000 : (time * 1_000_000).floor }
137
+ when :unixtime_nanos then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000_000 + time.nsec : (time * 1_000_000_000).floor }
135
138
  when :unixtime then ->(time){ time.to_i }
136
139
  else
137
140
  localtime = @inject_config.localtime && !@inject_config.utc
@@ -127,6 +127,10 @@ module Fluent
127
127
  nil
128
128
  end
129
129
 
130
+ def recalc_next_time
131
+ @next_time = calc_next_time
132
+ end
133
+
130
134
  def limit?
131
135
  if @forever
132
136
  false
@@ -199,7 +199,7 @@ module Fluent
199
199
  list = []
200
200
  data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
201
201
  if list.length == 0
202
- log.warn "cert_path does not contain a valid certificate"
202
+ raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
203
203
  end
204
204
  list
205
205
  end
@@ -53,11 +53,11 @@ module Fluent
53
53
  @enable_get_dump = config[:enable_get_dump]
54
54
  run_rpc_server
55
55
  end
56
- install_supervisor_signal_handlers
57
56
 
58
- if config[:signame]
59
- @signame = config[:signame]
57
+ if Fluent.windows?
60
58
  install_windows_event_handler
59
+ else
60
+ install_supervisor_signal_handlers
61
61
  end
62
62
 
63
63
  if counter = config[:counter_server]
@@ -70,6 +70,7 @@ module Fluent
70
70
  end
71
71
 
72
72
  def after_run
73
+ stop_windows_event_thread if Fluent.windows?
73
74
  stop_rpc_server if @rpc_endpoint
74
75
  stop_counter_server if @counter
75
76
  Fluent::Supervisor.cleanup_resources
@@ -92,7 +93,8 @@ module Fluent
92
93
  @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
93
94
  $log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
94
95
  if Fluent.windows?
95
- $log.warn "operation 'flushBuffersAndKillWorkers' is not supported on Windows now."
96
+ supervisor_sigusr1_handler
97
+ stop(true)
96
98
  else
97
99
  Process.kill :USR1, $$
98
100
  Process.kill :TERM, $$
@@ -101,7 +103,9 @@ module Fluent
101
103
  }
102
104
  @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
103
105
  $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
104
- unless Fluent.windows?
106
+ if Fluent.windows?
107
+ supervisor_sigusr1_handler
108
+ else
105
109
  Process.kill :USR1, $$
106
110
  end
107
111
  nil
@@ -125,7 +129,9 @@ module Fluent
125
129
 
126
130
  @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
127
131
  $log.debug "fluentd RPC got /api/config.gracefulReload request"
128
- unless Fluent.windows?
132
+ if Fluent.windows?
133
+ supervisor_sigusr2_handler
134
+ else
129
135
  Process.kill :USR2, $$
130
136
  end
131
137
 
@@ -159,38 +165,101 @@ module Fluent
159
165
  end
160
166
 
161
167
  def install_supervisor_signal_handlers
168
+ return if Fluent.windows?
169
+
162
170
  trap :HUP do
163
171
  $log.debug "fluentd supervisor process get SIGHUP"
164
172
  supervisor_sighup_handler
165
- end unless Fluent.windows?
173
+ end
166
174
 
167
175
  trap :USR1 do
168
176
  $log.debug "fluentd supervisor process get SIGUSR1"
169
177
  supervisor_sigusr1_handler
170
- end unless Fluent.windows?
178
+ end
171
179
 
172
180
  trap :USR2 do
173
181
  $log.debug 'fluentd supervisor process got SIGUSR2'
174
182
  supervisor_sigusr2_handler
175
- end unless Fluent.windows?
183
+ end
184
+ end
185
+
186
+ if Fluent.windows?
187
+ # Override some methods of ServerEngine::MultiSpawnWorker
188
+ # Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2
189
+ # handlers (see install_supervisor_signal_handlers), they should be
190
+ # disabled also on Windows, just send commands to workers instead.
191
+ def restart(graceful)
192
+ @monitors.each do |m|
193
+ m.send_command(graceful ? "GRACEFUL_RESTART\n" : "IMMEDIATE_RESTART\n")
194
+ end
195
+ end
196
+
197
+ def reload
198
+ @monitors.each do |m|
199
+ m.send_command("RELOAD\n")
200
+ end
201
+ end
176
202
  end
177
203
 
178
204
  def install_windows_event_handler
205
+ return unless Fluent.windows?
206
+
207
+ @pid_signame = "fluentd_#{$$}"
208
+ @signame = config[:signame]
209
+
179
210
  Thread.new do
180
- ev = Win32::Event.new(@signame)
211
+ ipc = Win32::Ipc.new(nil)
212
+ events = [
213
+ Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
214
+ Win32::Event.new("#{@pid_signame}"),
215
+ Win32::Event.new("#{@pid_signame}_HUP"),
216
+ Win32::Event.new("#{@pid_signame}_USR1"),
217
+ Win32::Event.new("#{@pid_signame}_USR2"),
218
+ ]
219
+ if @signame
220
+ signame_events = [
221
+ Win32::Event.new("#{@signame}"),
222
+ Win32::Event.new("#{@signame}_HUP"),
223
+ Win32::Event.new("#{@signame}_USR1"),
224
+ Win32::Event.new("#{@signame}_USR2"),
225
+ ]
226
+ events.concat(signame_events)
227
+ end
181
228
  begin
182
- ev.reset
183
- until WaitForSingleObject(ev.handle, 0) == WAIT_OBJECT_0
184
- sleep 1
229
+ loop do
230
+ idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
231
+ if idx > 0 && idx <= events.length
232
+ $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
233
+ else
234
+ $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
235
+ end
236
+ case idx
237
+ when 2, 6
238
+ stop(true)
239
+ when 3, 7
240
+ supervisor_sighup_handler
241
+ when 4, 8
242
+ supervisor_sigusr1_handler
243
+ when 5, 9
244
+ supervisor_sigusr2_handler
245
+ when 1
246
+ break
247
+ end
185
248
  end
186
- kill_worker
187
- stop(true)
188
249
  ensure
189
- ev.close
250
+ events.each { |event| event.close }
190
251
  end
191
252
  end
192
253
  end
193
254
 
255
+ def stop_windows_event_thread
256
+ if Fluent.windows?
257
+ ev = Win32::Event.open("#{@pid_signame}_STOP_EVENT_THREAD")
258
+ ev.set
259
+ ev.close
260
+ end
261
+ end
262
+
194
263
  def supervisor_sighup_handler
195
264
  kill_worker
196
265
  end
@@ -265,9 +334,25 @@ module Fluent
265
334
  def send_signal_to_workers(signal)
266
335
  return unless config[:worker_pid]
267
336
 
268
- config[:worker_pid].each_value do |pid|
269
- # don't rescue Errno::ESRCH here (invalid status)
270
- Process.kill(signal, pid)
337
+ if Fluent.windows?
338
+ send_command_to_workers(signal)
339
+ else
340
+ config[:worker_pid].each_value do |pid|
341
+ # don't rescue Errno::ESRCH here (invalid status)
342
+ Process.kill(signal, pid)
343
+ end
344
+ end
345
+ end
346
+
347
+ def send_command_to_workers(signal)
348
+ # Use SeverEngine's CommandSender on Windows
349
+ case signal
350
+ when :HUP
351
+ restart(false)
352
+ when :USR1
353
+ restart(true)
354
+ when :USR2
355
+ reload
271
356
  end
272
357
  end
273
358
  end
@@ -302,6 +387,7 @@ module Fluent
302
387
  log_level = params['log_level']
303
388
  suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
304
389
  ignore_repeated_log_interval = params['ignore_repeated_log_interval']
390
+ ignore_same_log_interval = params['ignore_same_log_interval']
305
391
 
306
392
  log_path = params['log_path']
307
393
  chuser = params['chuser']
@@ -309,7 +395,8 @@ module Fluent
309
395
  log_rotate_age = params['log_rotate_age']
310
396
  log_rotate_size = params['log_rotate_size']
311
397
 
312
- log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
398
+ log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
399
+ ignore_same_log_interval: ignore_same_log_interval}
313
400
  logger_initializer = Supervisor::LoggerInitializer.new(
314
401
  log_path, log_level, chuser, chgroup, log_opts,
315
402
  log_rotate_age: log_rotate_age,
@@ -347,6 +434,7 @@ module Fluent
347
434
  chumask: 0,
348
435
  suppress_repeated_stacktrace: suppress_repeated_stacktrace,
349
436
  ignore_repeated_log_interval: ignore_repeated_log_interval,
437
+ ignore_same_log_interval: ignore_same_log_interval,
350
438
  daemonize: daemonize,
351
439
  rpc_endpoint: params['rpc_endpoint'],
352
440
  counter_server: params['counter_server'],
@@ -441,10 +529,11 @@ module Fluent
441
529
  self
442
530
  end
443
531
 
444
- def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
532
+ def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
445
533
  $log.format = format if format
446
534
  $log.time_format = time_format if time_format
447
535
  $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
536
+ $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
448
537
 
449
538
  if @path && log_dir_perm
450
539
  File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -511,7 +600,8 @@ module Fluent
511
600
  @cl_opt = opt
512
601
  @conf = nil
513
602
 
514
- log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
603
+ log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
604
+ ignore_same_log_interval: opt[:ignore_same_log_interval]}
515
605
  @log = LoggerInitializer.new(
516
606
  @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
517
607
  log_rotate_age: @log_rotate_age,
@@ -635,7 +725,8 @@ module Fluent
635
725
  format: @system_config.log.format,
636
726
  time_format: @system_config.log.time_format,
637
727
  log_dir_perm: @system_config.dir_permission,
638
- ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
728
+ ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
729
+ ignore_same_log_interval: @system_config.ignore_same_log_interval
639
730
  )
640
731
 
641
732
  $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
@@ -740,33 +831,45 @@ module Fluent
740
831
  end
741
832
  end
742
833
 
743
- trap :USR1 do
744
- flush_buffer
745
- end unless Fluent.windows?
834
+ if Fluent.windows?
835
+ install_main_process_command_handlers
836
+ else
837
+ trap :USR1 do
838
+ flush_buffer
839
+ end
746
840
 
747
- trap :USR2 do
748
- reload_config
749
- end unless Fluent.windows?
841
+ trap :USR2 do
842
+ reload_config
843
+ end
844
+ end
845
+ end
750
846
 
751
- if Fluent.windows?
752
- command_pipe = STDIN.dup
753
- STDIN.reopen(File::NULL, "rb")
754
- command_pipe.binmode
755
- command_pipe.sync = true
847
+ def install_main_process_command_handlers
848
+ command_pipe = $stdin.dup
849
+ $stdin.reopen(File::NULL, "rb")
850
+ command_pipe.binmode
851
+ command_pipe.sync = true
756
852
 
757
- Thread.new do
758
- loop do
759
- cmd = command_pipe.gets.chomp
760
- case cmd
761
- when "GRACEFUL_STOP", "IMMEDIATE_STOP"
762
- $log.debug "fluentd main process get #{cmd} command"
763
- @finished = true
764
- $log.debug "getting start to shutdown main process"
765
- Fluent::Engine.stop
766
- break
767
- else
768
- $log.warn "fluentd main process get unknown command [#{cmd}]"
769
- end
853
+ Thread.new do
854
+ loop do
855
+ cmd = command_pipe.gets
856
+ break unless cmd
857
+
858
+ case cmd.chomp!
859
+ when "GRACEFUL_STOP", "IMMEDIATE_STOP"
860
+ $log.debug "fluentd main process get #{cmd} command"
861
+ @finished = true
862
+ $log.debug "getting start to shutdown main process"
863
+ Fluent::Engine.stop
864
+ break
865
+ when "GRACEFUL_RESTART"
866
+ $log.debug "fluentd main process get #{cmd} command"
867
+ flush_buffer
868
+ when "RELOAD"
869
+ $log.debug "fluentd main process get #{cmd} command"
870
+ reload_config
871
+ else
872
+ $log.warn "fluentd main process get unknown command [#{cmd}]"
770
873
  end
771
874
  end
772
875
  end
@@ -804,7 +907,7 @@ module Fluent
804
907
  Fluent::Engine.reload_config(conf)
805
908
  end
806
909
  rescue => e
807
- # it is guranteed that config file is valid by supervisor side. but it's not atomic becuase of using signals to commnicate between worker and super
910
+ # it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super
808
911
  # So need this rescue code
809
912
  $log.error("failed to reload config: #{e}")
810
913
  next