fluentd 1.14.4-x64-mingw32 → 1.15.0-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (74) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  3. data/.github/workflows/linux-test.yaml +1 -1
  4. data/.github/workflows/macos-test.yaml +5 -1
  5. data/.github/workflows/windows-test.yaml +9 -6
  6. data/CHANGELOG.md +115 -19
  7. data/CONTRIBUTING.md +1 -1
  8. data/MAINTAINERS.md +2 -2
  9. data/README.md +2 -23
  10. data/Rakefile +1 -1
  11. data/fluentd.gemspec +3 -1
  12. data/lib/fluent/command/ctl.rb +4 -1
  13. data/lib/fluent/command/fluentd.rb +14 -0
  14. data/lib/fluent/config/error.rb +12 -0
  15. data/lib/fluent/config/literal_parser.rb +2 -2
  16. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  17. data/lib/fluent/config/yaml_parser/loader.rb +91 -0
  18. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  19. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  20. data/lib/fluent/config/yaml_parser.rb +56 -0
  21. data/lib/fluent/config.rb +14 -1
  22. data/lib/fluent/event_router.rb +19 -1
  23. data/lib/fluent/plugin/bare_output.rb +1 -1
  24. data/lib/fluent/plugin/base.rb +1 -1
  25. data/lib/fluent/plugin/file_wrapper.rb +52 -107
  26. data/lib/fluent/plugin/in_forward.rb +1 -1
  27. data/lib/fluent/plugin/in_http.rb +11 -1
  28. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  29. data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
  30. data/lib/fluent/plugin/in_tail.rb +66 -47
  31. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  32. data/lib/fluent/plugin/output.rb +43 -33
  33. data/lib/fluent/plugin/parser.rb +3 -4
  34. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  35. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  36. data/lib/fluent/plugin_helper/server.rb +23 -4
  37. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  38. data/lib/fluent/plugin_helper/socket.rb +13 -2
  39. data/lib/fluent/registry.rb +2 -1
  40. data/lib/fluent/rpc.rb +4 -3
  41. data/lib/fluent/supervisor.rb +114 -27
  42. data/lib/fluent/system_config.rb +2 -1
  43. data/lib/fluent/version.rb +1 -1
  44. data/lib/fluent/winsvc.rb +2 -0
  45. data/test/command/test_ctl.rb +0 -1
  46. data/test/command/test_fluentd.rb +33 -0
  47. data/test/compat/test_parser.rb +1 -1
  48. data/test/config/test_system_config.rb +3 -1
  49. data/test/config/test_types.rb +1 -1
  50. data/test/plugin/in_tail/test_io_handler.rb +14 -4
  51. data/test/plugin/in_tail/test_position_file.rb +0 -63
  52. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  53. data/test/plugin/test_file_wrapper.rb +0 -68
  54. data/test/plugin/test_filter_parser.rb +1 -1
  55. data/test/plugin/test_filter_stdout.rb +2 -2
  56. data/test/plugin/test_in_forward.rb +0 -2
  57. data/test/plugin/test_in_http.rb +23 -0
  58. data/test/plugin/test_in_object_space.rb +9 -3
  59. data/test/plugin/test_in_syslog.rb +1 -1
  60. data/test/plugin/test_in_tail.rb +629 -353
  61. data/test/plugin/test_out_forward.rb +30 -20
  62. data/test/plugin/test_out_stdout.rb +2 -2
  63. data/test/plugin/test_output_as_buffered_retries.rb +53 -6
  64. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  65. data/test/plugin/test_parser_syslog.rb +1 -1
  66. data/test/plugin_helper/test_cert_option.rb +1 -1
  67. data/test/plugin_helper/test_child_process.rb +16 -4
  68. data/test/plugin_helper/test_retry_state.rb +602 -38
  69. data/test/plugin_helper/test_server.rb +18 -0
  70. data/test/test_config.rb +135 -4
  71. data/test/test_event_router.rb +17 -0
  72. data/test/test_formatter.rb +1 -1
  73. data/test/test_supervisor.rb +196 -6
  74. metadata +39 -5
@@ -44,6 +44,8 @@ module Fluent
44
44
 
45
45
  @timeout = timeout
46
46
  @timeout_at = @start + timeout
47
+ @has_reached_timeout = false
48
+ @has_timed_out = false
47
49
  @current = :primary
48
50
 
49
51
  if randomize_width < 0 || randomize_width > 0.5
@@ -98,7 +100,7 @@ module Fluent
98
100
  naive
99
101
  end
100
102
  elsif @current == :secondary
101
- naive = naive_next_time(@steps - @secondary_transition_steps + 1)
103
+ naive = naive_next_time(@steps - @secondary_transition_steps)
102
104
  if naive >= @timeout_at
103
105
  @timeout_at
104
106
  else
@@ -123,7 +125,15 @@ module Fluent
123
125
  @current = :secondary
124
126
  @secondary_transition_steps = @steps
125
127
  end
128
+
126
129
  @next_time = calc_next_time
130
+
131
+ if @has_reached_timeout
132
+ @has_timed_out = @next_time >= @timeout_at
133
+ else
134
+ @has_reached_timeout = @next_time >= @timeout_at
135
+ end
136
+
127
137
  nil
128
138
  end
129
139
 
@@ -135,7 +145,7 @@ module Fluent
135
145
  if @forever
136
146
  false
137
147
  else
138
- @next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps)
148
+ @has_timed_out || !!(@max_steps && @steps >= @max_steps)
139
149
  end
140
150
  end
141
151
  end
@@ -165,7 +175,7 @@ module Fluent
165
175
  end
166
176
 
167
177
  def calc_interval(num)
168
- interval = raw_interval(num - 1)
178
+ interval = raw_interval(num)
169
179
  if @max_interval && interval > @max_interval
170
180
  @max_interval
171
181
  else
@@ -175,7 +185,7 @@ module Fluent
175
185
  # Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.
176
186
  until interval.finite?
177
187
  num -= 1
178
- interval = raw_interval(num - 1)
188
+ interval = raw_interval(num)
179
189
  end
180
190
  interval
181
191
  end
@@ -80,8 +80,8 @@ module Fluent
80
80
  raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
81
81
  raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
82
82
 
83
- if proto == :tcp || proto == :tls # default linger_timeout only for server
84
- socket_options[:linger_timeout] ||= 0
83
+ if proto == :tcp || proto == :tls
84
+ socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
85
85
  end
86
86
 
87
87
  socket_option_validate!(proto, **socket_options)
@@ -132,8 +132,8 @@ module Fluent
132
132
  raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
133
133
  raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
134
134
 
135
- if proto == :tcp || proto == :tls # default linger_timeout only for server
136
- socket_options[:linger_timeout] ||= 0
135
+ if proto == :tcp || proto == :tls
136
+ socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
137
137
  end
138
138
 
139
139
  unless socket
@@ -263,6 +263,25 @@ module Fluent
263
263
  include Fluent::Configurable
264
264
  config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
265
265
  config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
266
+
267
+ ### Socket Params ###
268
+
269
+ # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
270
+ # Set positive value if needing to send FIN on closing on non-Windows.
271
+ # (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
272
+ # See `socket_option.rb`.)
273
+ # NOTE:
274
+ # Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.
275
+ # This is because there is no positive reason to do so.
276
+ # `linger_timeout` option in particular needs to be defined here
277
+ # although it can be specified from each plugin as well.
278
+ # This is because this helper fixes the default value to `0` for its own reason
279
+ # and it has a critical effect on the behavior.
280
+ desc 'The timeout time used to set linger option.'
281
+ config_param :linger_timeout, :integer, default: 0
282
+
283
+ ### TLS Params ###
284
+
266
285
  config_param :version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
267
286
  config_param :min_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
268
287
  config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
@@ -44,7 +44,7 @@ module Fluent
44
44
 
45
45
  @discovery_manager.start
46
46
  unless @discovery_manager.static_config?
47
- timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_iterval) do
47
+ timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_interval) do
48
48
  @discovery_manager.run_once
49
49
  end
50
50
  end
@@ -96,7 +96,7 @@ module Fluent
96
96
  # @param custom_build_method [Proc]
97
97
  def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
98
98
  @_plugin_helper_service_discovery_title = title
99
- @_plugin_helper_service_discovery_iterval = interval
99
+ @_plugin_helper_service_discovery_interval = interval
100
100
 
101
101
  @discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
102
102
  log: log,
@@ -96,6 +96,7 @@ module Fluent
96
96
  enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
97
97
  cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
98
98
  cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
99
+ connect_timeout: nil,
99
100
  **kwargs, &block)
100
101
 
101
102
  host_is_ipaddress = IPAddr.new(host) rescue false
@@ -158,13 +159,23 @@ module Fluent
158
159
  end
159
160
  Fluent::TLS.set_version_to_context(context, version, min_version, max_version)
160
161
 
161
- tcpsock = socket_create_tcp(host, port, **kwargs)
162
+ tcpsock = socket_create_tcp(host, port, connect_timeout: connect_timeout, **kwargs)
162
163
  sock = WrappedSocket::TLS.new(tcpsock, context)
163
164
  sock.sync_close = true
164
165
  sock.hostname = fqdn if verify_fqdn && fqdn && sock.respond_to?(:hostname=)
165
166
 
166
167
  log.trace "entering TLS handshake"
167
- sock.connect
168
+ if connect_timeout
169
+ begin
170
+ Timeout.timeout(connect_timeout) { sock.connect }
171
+ rescue Timeout::Error
172
+ log.warn "timeout while connecting tls session", host: host
173
+ sock.close rescue nil
174
+ raise
175
+ end
176
+ else
177
+ sock.connect
178
+ end
168
179
 
169
180
  begin
170
181
  if verify_fqdn
@@ -45,7 +45,8 @@ module Fluent
45
45
  if value = @map[type]
46
46
  return value
47
47
  end
48
- raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins" # TODO error class
48
+ raise NotFoundPluginError.new("Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins",
49
+ kind: @kind, type: type)
49
50
  end
50
51
 
51
52
  def reverse_lookup(value)
data/lib/fluent/rpc.rb CHANGED
@@ -20,9 +20,10 @@ module Fluent
20
20
  module RPC
21
21
  class Server
22
22
  def initialize(endpoint, log)
23
- bind, port = endpoint.split(':')
24
- @bind = bind
25
- @port = port
23
+ m = endpoint.match(/^\[?(?<host>[0-9a-zA-Z:\-\.]+)\]?:(?<port>[0-9]+)$/)
24
+ raise Fluent::ConfigError, "Invalid rpc_endpoint: #{endpoint}" unless m
25
+ @bind = m[:host]
26
+ @port = m[:port]
26
27
  @log = log
27
28
 
28
29
  @server = WEBrick::HTTPServer.new(
@@ -16,6 +16,7 @@
16
16
 
17
17
  require 'fileutils'
18
18
  require 'open3'
19
+ require 'pathname'
19
20
 
20
21
  require 'fluent/config'
21
22
  require 'fluent/counter'
@@ -215,44 +216,50 @@ module Fluent
215
216
  Thread.new do
216
217
  ipc = Win32::Ipc.new(nil)
217
218
  events = [
218
- Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
219
- Win32::Event.new("#{@pid_signame}"),
220
- Win32::Event.new("#{@pid_signame}_HUP"),
221
- Win32::Event.new("#{@pid_signame}_USR1"),
222
- Win32::Event.new("#{@pid_signame}_USR2"),
219
+ {win32_event: Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"), action: :stop_event_thread},
220
+ {win32_event: Win32::Event.new("#{@pid_signame}"), action: :stop},
221
+ {win32_event: Win32::Event.new("#{@pid_signame}_HUP"), action: :hup},
222
+ {win32_event: Win32::Event.new("#{@pid_signame}_USR1"), action: :usr1},
223
+ {win32_event: Win32::Event.new("#{@pid_signame}_USR2"), action: :usr2},
224
+ {win32_event: Win32::Event.new("#{@pid_signame}_CONT"), action: :cont},
223
225
  ]
224
226
  if @signame
225
227
  signame_events = [
226
- Win32::Event.new("#{@signame}"),
227
- Win32::Event.new("#{@signame}_HUP"),
228
- Win32::Event.new("#{@signame}_USR1"),
229
- Win32::Event.new("#{@signame}_USR2"),
228
+ {win32_event: Win32::Event.new("#{@signame}"), action: :stop},
229
+ {win32_event: Win32::Event.new("#{@signame}_HUP"), action: :hup},
230
+ {win32_event: Win32::Event.new("#{@signame}_USR1"), action: :usr1},
231
+ {win32_event: Win32::Event.new("#{@signame}_USR2"), action: :usr2},
232
+ {win32_event: Win32::Event.new("#{@signame}_CONT"), action: :cont},
230
233
  ]
231
234
  events.concat(signame_events)
232
235
  end
233
236
  begin
234
237
  loop do
235
- idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
236
- if idx > 0 && idx <= events.length
237
- $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
238
+ ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, Windows::Synchronize::INFINITE)
239
+ event_idx = ipc_idx - 1
240
+
241
+ if event_idx >= 0 && event_idx < events.length
242
+ $log.debug("Got Win32 event \"#{events[event_idx][:win32_event].name}\"")
238
243
  else
239
- $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
244
+ $log.warn("Unexpected return value of Win32::Ipc#wait_any: #{ipc_idx}")
240
245
  end
241
- case idx
242
- when 2, 6
246
+ case events[event_idx][:action]
247
+ when :stop
243
248
  stop(true)
244
- when 3, 7
249
+ when :hup
245
250
  supervisor_sighup_handler
246
- when 4, 8
251
+ when :usr1
247
252
  supervisor_sigusr1_handler
248
- when 5, 9
253
+ when :usr2
249
254
  supervisor_sigusr2_handler
250
- when 1
255
+ when :cont
256
+ supervisor_dump_handler_for_windows
257
+ when :stop_event_thread
251
258
  break
252
259
  end
253
260
  end
254
261
  ensure
255
- events.each { |event| event.close }
262
+ events.each { |event| event[:win32_event].close }
256
263
  end
257
264
  end
258
265
  end
@@ -302,6 +309,26 @@ module Fluent
302
309
  $log.error "Failed to reload config file: #{e}"
303
310
  end
304
311
 
312
+ def supervisor_dump_handler_for_windows
313
+ # As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
314
+ # and it is implemented before the implementation of the function for Windows.
315
+ # It is possible to trap SIGCONT and handle it here also on UNIX-like,
316
+ # but for backward compatibility, this handler is currently for a Windows-only.
317
+ raise "[BUG] This function is for Windows ONLY." unless Fluent.windows?
318
+
319
+ Thread.new do
320
+ begin
321
+ FluentSigdump.dump_windows
322
+ rescue => e
323
+ $log.error "failed to dump: #{e}"
324
+ end
325
+ end
326
+
327
+ send_signal_to_workers(:CONT)
328
+ rescue => e
329
+ $log.error "failed to dump: #{e}"
330
+ end
331
+
305
332
  def kill_worker
306
333
  if config[:worker_pid]
307
334
  pids = config[:worker_pid].clone
@@ -358,6 +385,14 @@ module Fluent
358
385
  restart(true)
359
386
  when :USR2
360
387
  reload
388
+ when :CONT
389
+ dump_all_windows_workers
390
+ end
391
+ end
392
+
393
+ def dump_all_windows_workers
394
+ @monitors.each do |m|
395
+ m.send_command("DUMP\n")
361
396
  end
362
397
  end
363
398
  end
@@ -397,6 +432,7 @@ module Fluent
397
432
  log_path = params['log_path']
398
433
  chuser = params['chuser']
399
434
  chgroup = params['chgroup']
435
+ chumask = params['chumask']
400
436
  log_rotate_age = params['log_rotate_age']
401
437
  log_rotate_size = params['log_rotate_size']
402
438
 
@@ -436,7 +472,7 @@ module Fluent
436
472
  logger_initializer: logger_initializer,
437
473
  chuser: chuser,
438
474
  chgroup: chgroup,
439
- chumask: 0,
475
+ chumask: chumask,
440
476
  suppress_repeated_stacktrace: suppress_repeated_stacktrace,
441
477
  ignore_repeated_log_interval: ignore_repeated_log_interval,
442
478
  ignore_same_log_interval: ignore_same_log_interval,
@@ -457,7 +493,8 @@ module Fluent
457
493
  config_path: path,
458
494
  main_cmd: params['main_cmd'],
459
495
  signame: params['signame'],
460
- disable_shared_socket: params['disable_shared_socket']
496
+ disable_shared_socket: params['disable_shared_socket'],
497
+ restart_worker_interval: params['restart_worker_interval'],
461
498
  }
462
499
  if daemonize
463
500
  se_config[:pid_path] = pid_path
@@ -565,6 +602,7 @@ module Fluent
565
602
  setup_path: nil,
566
603
  chuser: nil,
567
604
  chgroup: nil,
605
+ chumask: "0",
568
606
  root_dir: nil,
569
607
  suppress_interval: 0,
570
608
  suppress_repeated_stacktrace: true,
@@ -578,7 +616,8 @@ module Fluent
578
616
  standalone_worker: false,
579
617
  signame: nil,
580
618
  conf_encoding: 'utf-8',
581
- disable_shared_socket: nil
619
+ disable_shared_socket: nil,
620
+ config_file_type: :guess,
582
621
  }
583
622
  end
584
623
 
@@ -591,6 +630,7 @@ module Fluent
591
630
  end
592
631
 
593
632
  def initialize(opt)
633
+ @config_file_type = opt[:config_file_type]
594
634
  @daemonize = opt[:daemonize]
595
635
  @standalone_worker= opt[:standalone_worker]
596
636
  @config_path = opt[:config_path]
@@ -603,6 +643,7 @@ module Fluent
603
643
  @plugin_dirs = opt[:plugin_dirs]
604
644
  @chgroup = opt[:chgroup]
605
645
  @chuser = opt[:chuser]
646
+ @chumask = opt[:chumask]
606
647
 
607
648
  @log_rotate_age = opt[:log_rotate_age]
608
649
  @log_rotate_size = opt[:log_rotate_size]
@@ -615,7 +656,9 @@ module Fluent
615
656
  @conf = Fluent::Config.build(config_path: @config_path,
616
657
  encoding: @conf_encoding ? @conf_encoding : 'utf-8',
617
658
  additional_config: @inline_config ? @inline_config : nil,
618
- use_v1_config: !!@use_v1_config)
659
+ use_v1_config: !!@use_v1_config,
660
+ type: @config_file_type,
661
+ )
619
662
  @system_config = build_system_config(@conf)
620
663
  if @system_config.log
621
664
  @log_rotate_age ||= @system_config.log.rotate_age
@@ -709,7 +752,7 @@ module Fluent
709
752
  create_socket_manager if @standalone_worker
710
753
  if @standalone_worker
711
754
  ServerEngine::Privilege.change(@chuser, @chgroup)
712
- File.umask(0)
755
+ File.umask(@chumask.to_i(8))
713
756
  end
714
757
  MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
715
758
  Fluent::Engine.init(@system_config)
@@ -741,7 +784,13 @@ module Fluent
741
784
  $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
742
785
  @inline_config = STDIN.read
743
786
  end
744
- @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config)
787
+ @conf = Fluent::Config.build(
788
+ config_path: @config_path,
789
+ encoding: @conf_encoding,
790
+ additional_config: @inline_config,
791
+ use_v1_config: @use_v1_config,
792
+ type: @config_file_type,
793
+ )
745
794
  @system_config = build_system_config(@conf)
746
795
 
747
796
  @log.level = @system_config.log_level
@@ -819,7 +868,8 @@ module Fluent
819
868
  'counter_server' => @system_config.counter_server,
820
869
  'log_format' => @system_config.log.format,
821
870
  'log_time_format' => @system_config.log.time_format,
822
- 'disable_shared_socket' => @system_config.disable_shared_socket
871
+ 'disable_shared_socket' => @system_config.disable_shared_socket,
872
+ 'restart_worker_interval' => @system_config.restart_worker_interval,
823
873
  }
824
874
 
825
875
  se = ServerEngine.create(ServerModule, WorkerModule){
@@ -893,6 +943,9 @@ module Fluent
893
943
  when "RELOAD"
894
944
  $log.debug "fluentd main process get #{cmd} command"
895
945
  reload_config
946
+ when "DUMP"
947
+ $log.debug "fluentd main process get #{cmd} command"
948
+ dump
896
949
  else
897
950
  $log.warn "fluentd main process get unknown command [#{cmd}]"
898
951
  end
@@ -926,6 +979,7 @@ module Fluent
926
979
  encoding: @conf_encoding,
927
980
  additional_config: @inline_config,
928
981
  use_v1_config: @use_v1_config,
982
+ type: @config_file_type,
929
983
  )
930
984
 
931
985
  Fluent::VariableStore.try_to_reset do
@@ -942,6 +996,16 @@ module Fluent
942
996
  end
943
997
  end
944
998
 
999
+ def dump
1000
+ Thread.new do
1001
+ begin
1002
+ FluentSigdump.dump_windows
1003
+ rescue => e
1004
+ $log.error("failed to dump: #{e}")
1005
+ end
1006
+ end
1007
+ end
1008
+
945
1009
  def logging_with_console_output
946
1010
  yield $log
947
1011
  unless @log.stdout?
@@ -1051,4 +1115,27 @@ module Fluent
1051
1115
  fluentd_spawn_cmd
1052
1116
  end
1053
1117
  end
1118
+
1119
+ module FluentSigdump
1120
+ def self.dump_windows
1121
+ raise "[BUG] WindowsSigdump::dump is for Windows ONLY." unless Fluent.windows?
1122
+
1123
+ # Sigdump outputs under `/tmp` dir without `SIGDUMP_PATH` specified,
1124
+ # but `/tmp` dir may not exist on Windows by default.
1125
+ # So use the systemroot-temp-dir instead.
1126
+ dump_filepath = ENV['SIGDUMP_PATH'].nil? || ENV['SIGDUMP_PATH'].empty? \
1127
+ ? "#{ENV['windir']}/Temp/fluentd-sigdump-#{Process.pid}.log"
1128
+ : get_path_with_pid(ENV['SIGDUMP_PATH'])
1129
+
1130
+ require 'sigdump'
1131
+ Sigdump.dump(dump_filepath)
1132
+
1133
+ $log.info "dump to #{dump_filepath}."
1134
+ end
1135
+
1136
+ def self.get_path_with_pid(raw_path)
1137
+ path = Pathname.new(raw_path)
1138
+ path.sub_ext("-#{Process.pid}#{path.extname}").to_s
1139
+ end
1140
+ end
1054
1141
  end
@@ -22,7 +22,7 @@ module Fluent
22
22
  include Configurable
23
23
 
24
24
  SYSTEM_CONFIG_PARAMETERS = [
25
- :workers, :root_dir, :log_level,
25
+ :workers, :restart_worker_interval, :root_dir, :log_level,
26
26
  :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
27
27
  :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
28
28
  :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
@@ -32,6 +32,7 @@ module Fluent
32
32
  ]
33
33
 
34
34
  config_param :workers, :integer, default: 1
35
+ config_param :restart_worker_interval, :time, default: 0
35
36
  config_param :root_dir, :string, default: nil
36
37
  config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
37
38
  config_param :suppress_repeated_stacktrace, :bool, default: nil
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.14.4'
19
+ VERSION = '1.15.0'
20
20
 
21
21
  end
data/lib/fluent/winsvc.rb CHANGED
@@ -84,6 +84,8 @@ begin
84
84
  set_event("#{@service_name}_HUP")
85
85
  when 129
86
86
  set_event("#{@service_name}_USR1")
87
+ when 130
88
+ set_event("#{@service_name}_CONT")
87
89
  end
88
90
  end
89
91
 
@@ -7,7 +7,6 @@ require 'fluent/command/ctl'
7
7
 
8
8
  class TestFluentdCtl < ::Test::Unit::TestCase
9
9
  def assert_win32_event(event_name, command, pid_or_svcname)
10
- command, event_suffix = data
11
10
  event = Win32::Event.new(event_name)
12
11
  ipc = Win32::Ipc.new(event.handle)
13
12
  ret = Win32::Ipc::TIMEOUT
@@ -589,6 +589,39 @@ CONF
589
589
  )
590
590
  end
591
591
 
592
+ sub_test_case "YAML config format" do
593
+ test 'success to start the number of workers specified in configuration' do
594
+ conf = <<'CONF'
595
+ system:
596
+ workers: 2
597
+ root_dir: "#{@root_path}"
598
+ config:
599
+ - source:
600
+ $type: dummy
601
+ $id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
602
+ $label: '@dummydata'
603
+ tag: dummy
604
+ dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
605
+
606
+ - label:
607
+ $name: '@dummydata'
608
+ config:
609
+ - match:
610
+ $tag: dummy
611
+ $type: "null"
612
+ $id: blackhole
613
+ CONF
614
+ conf_path = create_conf_file('workers1.yaml', conf)
615
+ assert Dir.exist?(@root_path)
616
+
617
+ assert_log_matches(
618
+ create_cmdline(conf_path),
619
+ "#0 fluentd worker is now running worker=0",
620
+ "#1 fluentd worker is now running worker=1"
621
+ )
622
+ end
623
+ end
624
+
592
625
  test 'success to start the number of workers specified by command line option' do
593
626
  conf = <<CONF
594
627
  <system>
@@ -22,7 +22,7 @@ class TextParserTest < ::Test::Unit::TestCase
22
22
  Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
23
23
 
24
24
  def test_lookup_unknown_format
25
- assert_raise Fluent::ConfigError do
25
+ assert_raise Fluent::NotFoundPluginError do
26
26
  Fluent::Plugin.new_parser('unknown')
27
27
  end
28
28
  end
@@ -19,6 +19,7 @@ module Fluent::Config
19
19
  @system_config = nil
20
20
  @cl_opt = {
21
21
  wokers: nil,
22
+ restart_worker_interval: nil,
22
23
  root_dir: nil,
23
24
  log: FakeLoggerInitializer.new,
24
25
  log_level: Fluent::Log::LEVEL_INFO,
@@ -72,6 +73,7 @@ module Fluent::Config
72
73
  sc = Fluent::SystemConfig.new(conf)
73
74
  sc.overwrite_variables(**s.for_system_config)
74
75
  assert_equal(1, sc.workers)
76
+ assert_equal(0, sc.restart_worker_interval)
75
77
  assert_nil(sc.root_dir)
76
78
  assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
77
79
  assert_nil(sc.suppress_repeated_stacktrace)
@@ -88,6 +90,7 @@ module Fluent::Config
88
90
 
89
91
  data(
90
92
  'workers' => ['workers', 3],
93
+ 'restart_worker_interval' => ['restart_worker_interval', 60],
91
94
  'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
92
95
  'log_level' => ['log_level', 'error'],
93
96
  'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
@@ -174,7 +177,6 @@ module Fluent::Config
174
177
  </log>
175
178
  </system>
176
179
  EOS
177
- s = FakeSupervisor.new
178
180
  sc = Fluent::SystemConfig.new(conf)
179
181
  assert_equal(3, sc.log.rotate_age)
180
182
  end
@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
190
190
  data("val" => [:val, 'val'],
191
191
  "v" => [:v, 'v'],
192
192
  "value" => [:value, 'value'])
193
- test 'enum' do |(expected, val, list)|
193
+ test 'enum' do |(expected, val)|
194
194
  assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
195
195
  end
196
196
 
@@ -21,12 +21,20 @@ class IntailIOHandlerTest < Test::Unit::TestCase
21
21
  @file.unlink rescue nil
22
22
  end
23
23
 
24
+ def create_target_info
25
+ Fluent::Plugin::TailInput::TargetInfo.new(@file.path, Fluent::FileWrapper.stat(@file.path).ino)
26
+ end
27
+
28
+ def create_watcher
29
+ Fluent::Plugin::TailInput::TailWatcher.new(create_target_info, nil, nil, nil, nil, nil, nil, nil, nil)
30
+ end
31
+
24
32
  test '#on_notify load file content and passed it to receive_lines method' do
25
33
  text = "this line is test\ntest line is test\n"
26
34
  @file.write(text)
27
35
  @file.close
28
36
 
29
- watcher = 'watcher'
37
+ watcher = create_watcher
30
38
 
31
39
  update_pos = 0
32
40
 
@@ -61,7 +69,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
61
69
 
62
70
  update_pos = 0
63
71
 
64
- watcher = 'watcher'
72
+ watcher = create_watcher
65
73
  stub(watcher).pe do
66
74
  pe = 'position_file'
67
75
  stub(pe).read_pos { 0 }
@@ -92,7 +100,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
92
100
 
93
101
  update_pos = 0
94
102
 
95
- watcher = 'watcher'
103
+ watcher = create_watcher
96
104
  stub(watcher).pe do
97
105
  pe = 'position_file'
98
106
  stub(pe).read_pos { 0 }
@@ -107,6 +115,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
107
115
  end
108
116
 
109
117
  r.on_notify
118
+ assert_equal text.bytesize, update_pos
110
119
  assert_equal 8, returned_lines[0].size
111
120
  end
112
121
 
@@ -118,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
118
127
 
119
128
  update_pos = 0
120
129
 
121
- watcher = 'watcher'
130
+ watcher = create_watcher
122
131
  stub(watcher).pe do
123
132
  pe = 'position_file'
124
133
  stub(pe).read_pos { 0 }
@@ -133,6 +142,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
133
142
  end
134
143
 
135
144
  r.on_notify
145
+ assert_equal text.bytesize, update_pos
136
146
  assert_equal 5, returned_lines[0].size
137
147
  assert_equal 3, returned_lines[1].size
138
148
  end