fluentd 0.14.10-x86-mingw32 → 0.14.11-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +14 -6
  3. data/ChangeLog +28 -2
  4. data/appveyor.yml +1 -0
  5. data/lib/fluent/engine.rb +4 -7
  6. data/lib/fluent/error.rb +30 -0
  7. data/lib/fluent/log.rb +0 -7
  8. data/lib/fluent/plugin/base.rb +11 -0
  9. data/lib/fluent/plugin/buf_file.rb +9 -7
  10. data/lib/fluent/plugin/formatter_csv.rb +4 -2
  11. data/lib/fluent/plugin/in_forward.rb +46 -17
  12. data/lib/fluent/plugin/in_http.rb +2 -0
  13. data/lib/fluent/plugin/in_monitor_agent.rb +27 -2
  14. data/lib/fluent/plugin/in_syslog.rb +52 -36
  15. data/lib/fluent/plugin/in_tail.rb +1 -0
  16. data/lib/fluent/plugin/out_forward.rb +39 -29
  17. data/lib/fluent/plugin/output.rb +17 -0
  18. data/lib/fluent/plugin/storage_local.rb +16 -13
  19. data/lib/fluent/plugin_helper/storage.rb +21 -9
  20. data/lib/fluent/plugin_id.rb +17 -0
  21. data/lib/fluent/supervisor.rb +73 -45
  22. data/lib/fluent/system_config.rb +24 -21
  23. data/lib/fluent/version.rb +1 -1
  24. data/test/command/test_fluentd.rb +348 -0
  25. data/test/config/test_system_config.rb +39 -31
  26. data/test/plugin/test_base.rb +20 -0
  27. data/test/plugin/test_buf_file.rb +40 -0
  28. data/test/plugin/test_formatter_csv.rb +8 -0
  29. data/test/plugin/test_in_forward.rb +56 -21
  30. data/test/plugin/test_in_monitor_agent.rb +80 -8
  31. data/test/plugin/test_in_syslog.rb +75 -45
  32. data/test/plugin/test_out_file.rb +0 -1
  33. data/test/plugin/test_out_forward.rb +19 -11
  34. data/test/plugin/test_output.rb +44 -0
  35. data/test/plugin/test_storage_local.rb +290 -2
  36. data/test/plugin_helper/test_child_process.rb +40 -39
  37. data/test/plugin_helper/test_storage.rb +4 -3
  38. data/test/test_log.rb +1 -1
  39. data/test/test_output.rb +3 -0
  40. data/test/test_plugin_id.rb +101 -0
  41. data/test/test_supervisor.rb +3 -0
  42. metadata +7 -2
@@ -30,6 +30,10 @@ module Fluent
30
30
  StorageState = Struct.new(:storage, :running)
31
31
 
32
32
  def storage_create(usage: '', type: nil, conf: nil, default_type: nil)
33
+ if conf && !conf.arg.empty?
34
+ usage = conf.arg
35
+ end
36
+
33
37
  s = @_storages[usage]
34
38
  if s && s.running
35
39
  return s.storage
@@ -72,7 +76,7 @@ module Fluent
72
76
  module StorageParams
73
77
  include Fluent::Configurable
74
78
  # minimum section definition to instantiate storage plugin instances
75
- config_section :storage, required: false, multi: true, param_name: :storage_configs do
79
+ config_section :storage, required: false, multi: true, param_name: :storage_configs, init: true do
76
80
  config_argument :usage, :string, default: ''
77
81
  config_param :@type, :string, default: Fluent::Plugin::Storage::DEFAULT_TYPE
78
82
  end
@@ -194,6 +198,10 @@ module Fluent
194
198
  def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
195
199
  def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
196
200
 
201
+ def method_missing(name, *args)
202
+ @monitor.synchronize{ @storage.__send__(name, *args) }
203
+ end
204
+
197
205
  def persistent_always?
198
206
  true
199
207
  end
@@ -274,7 +282,7 @@ module Fluent
274
282
 
275
283
  def initialize(storage)
276
284
  @storage = storage
277
- @mutex = Mutex.new
285
+ @monitor = Monitor.new
278
286
  end
279
287
 
280
288
  def_delegators :@storage, :persistent, :autosave, :autosave_interval, :save_at_shutdown
@@ -282,6 +290,10 @@ module Fluent
282
290
  def_delegators :@storage, :start, :stop, :before_shutdown, :shutdown, :after_shutdown, :close, :terminate
283
291
  def_delegators :@storage, :started?, :stopped?, :before_shutdown?, :shutdown?, :after_shutdown?, :closed?, :terminated?
284
292
 
293
+ def method_missing(name, *args)
294
+ @monitor.synchronize{ @storage.__send__(name, *args) }
295
+ end
296
+
285
297
  def synchronized?
286
298
  true
287
299
  end
@@ -291,35 +303,35 @@ module Fluent
291
303
  end
292
304
 
293
305
  def load
294
- @mutex.synchronize do
306
+ @monitor.synchronize do
295
307
  @storage.load
296
308
  end
297
309
  end
298
310
 
299
311
  def save
300
- @mutex.synchronize do
312
+ @monitor.synchronize do
301
313
  @storage.save
302
314
  end
303
315
  end
304
316
 
305
317
  def get(key)
306
- @mutex.synchronize{ @storage.get(key) }
318
+ @monitor.synchronize{ @storage.get(key) }
307
319
  end
308
320
 
309
321
  def fetch(key, defval)
310
- @mutex.synchronize{ @storage.fetch(key, defval) }
322
+ @monitor.synchronize{ @storage.fetch(key, defval) }
311
323
  end
312
324
 
313
325
  def put(key, value)
314
- @mutex.synchronize{ @storage.put(key, value) }
326
+ @monitor.synchronize{ @storage.put(key, value) }
315
327
  end
316
328
 
317
329
  def delete(key)
318
- @mutex.synchronize{ @storage.delete(key) }
330
+ @monitor.synchronize{ @storage.delete(key) }
319
331
  end
320
332
 
321
333
  def update(key, &block)
322
- @mutex.synchronize do
334
+ @monitor.synchronize do
323
335
  v = block.call(@storage.get(key))
324
336
  @storage.put(key, v)
325
337
  v
@@ -20,6 +20,11 @@ module Fluent
20
20
  module PluginId
21
21
  @@configured_ids = Set.new
22
22
 
23
+ def initialize
24
+ super
25
+ @_plugin_root_dir = nil
26
+ end
27
+
23
28
  def configure(conf)
24
29
  @id = conf['@id']
25
30
  @_id_configured = !!@id # plugin id is explicitly configured by users (or not)
@@ -59,5 +64,17 @@ module Fluent
59
64
  "object:#{object_id.to_s(16)}"
60
65
  end
61
66
  end
67
+
68
+ def plugin_root_dir
69
+ return @_plugin_root_dir if @_plugin_root_dir
70
+ return nil unless system_config.root_dir
71
+ return nil unless plugin_id_configured?
72
+
73
+ # Fluent::Plugin::Base#fluentd_worker_id
74
+ dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
75
+ FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
76
+ @_plugin_root_dir = dir.freeze
77
+ dir
78
+ end
62
79
  end
63
80
  end
@@ -16,10 +16,12 @@
16
16
 
17
17
  require 'etc'
18
18
  require 'fcntl'
19
+ require 'fileutils'
19
20
 
20
21
  require 'fluent/config'
21
22
  require 'fluent/env'
22
23
  require 'fluent/engine'
24
+ require 'fluent/error'
23
25
  require 'fluent/log'
24
26
  require 'fluent/plugin'
25
27
  require 'fluent/rpc'
@@ -191,7 +193,10 @@ module Fluent
191
193
  module WorkerModule
192
194
  def spawn(process_manager)
193
195
  main_cmd = config[:main_cmd]
194
- @pm = process_manager.spawn(main_cmd)
196
+ env = {
197
+ 'SERVERENGINE_WORKER_ID' => @worker_id.to_i.to_s,
198
+ }
199
+ @pm = process_manager.spawn(env, *main_cmd)
195
200
  end
196
201
 
197
202
  def after_start
@@ -225,6 +230,7 @@ module Fluent
225
230
  fluentd_conf = Fluent::Config.parse(config_data, config_fname, config_basedir, params['use_v1_config'])
226
231
  system_config = SystemConfig.create(fluentd_conf)
227
232
 
233
+ root_dir = system_config.root_dir || params['root_dir']
228
234
  log_level = system_config.log_level || params['log_level']
229
235
  suppress_repeated_stacktrace = system_config.suppress_repeated_stacktrace || params['suppress_repeated_stacktrace']
230
236
  log_path = params['log_path']
@@ -263,6 +269,7 @@ module Fluent
263
269
  auto_heartbeat: false,
264
270
  unrecoverable_exit_codes: [2],
265
271
  stop_immediately_at_unrecoverable_exit: true,
272
+ root_dir: root_dir,
266
273
  logger: logger,
267
274
  log: logger.out,
268
275
  log_path: log_path,
@@ -364,6 +371,7 @@ module Fluent
364
371
  setup_path: nil,
365
372
  chuser: nil,
366
373
  chgroup: nil,
374
+ root_dir: nil,
367
375
  suppress_interval: 0,
368
376
  suppress_repeated_stacktrace: true,
369
377
  without_source: false,
@@ -392,6 +400,7 @@ module Fluent
392
400
  @rpc_server = nil
393
401
  @process_name = nil
394
402
 
403
+ @root_dir = opt[:root_dir]
395
404
  @log_level = opt[:log_level]
396
405
  @log_rotate_age = opt[:log_rotate_age]
397
406
  @log_rotate_size = opt[:log_rotate_size]
@@ -416,6 +425,20 @@ module Fluent
416
425
  read_config
417
426
  set_system_config
418
427
 
428
+ if @root_dir
429
+ if File.exist?(@root_dir)
430
+ unless Dir.exist?(@root_dir)
431
+ raise Fluent::InvalidRootDirectory, "non directory entry exists:#{@root_dir}"
432
+ end
433
+ else
434
+ begin
435
+ FileUtils.mkdir_p(@root_dir)
436
+ rescue => e
437
+ raise Fluent::InvalidRootDirectory, "failed to create root directory:#{@root_dir}, #{e.inspect}"
438
+ end
439
+ end
440
+ end
441
+
419
442
  dry_run if @dry_run
420
443
  supervise
421
444
  end
@@ -425,7 +448,8 @@ module Fluent
425
448
  'config_path' => @config_path,
426
449
  'pid_file' => @daemonize,
427
450
  'plugin_dirs' => @plugin_dirs,
428
- 'log_path' => @log_path
451
+ 'log_path' => @log_path,
452
+ 'root_dir' => @root_dir,
429
453
  }
430
454
  end
431
455
 
@@ -444,7 +468,7 @@ module Fluent
444
468
 
445
469
  install_main_process_signal_handlers
446
470
 
447
- $log.info "starting fluentd-#{Fluent::VERSION} without supervision"
471
+ $log.info "starting fluentd-#{Fluent::VERSION} without supervision", pid: Process.pid
448
472
 
449
473
  main_process do
450
474
  create_socket_manager if @standalone_worker
@@ -501,28 +525,17 @@ module Fluent
501
525
  end
502
526
 
503
527
  def supervise
504
- $log.info "starting fluentd-#{Fluent::VERSION}"
528
+ Process.setproctitle("supervisor:#{@process_name}") if @process_name
529
+ $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid
505
530
 
506
531
  rubyopt = ENV["RUBYOPT"]
507
- if Fluent.windows?
508
- # Shellwords doesn't work on windows, then used gsub for adapting space char instead of Shellwords
509
- fluentd_spawn_cmd = ServerEngine.ruby_bin_path + " -Eascii-8bit:ascii-8bit "
510
- fluentd_spawn_cmd << ' "' + rubyopt.gsub('"', '""') + '" ' if rubyopt
511
- fluentd_spawn_cmd << ' "' + $0.gsub('"', '""') + '" '
512
- $fluentdargv.each{|a|
513
- fluentd_spawn_cmd << ('"' + a.gsub('"', '""') + '" ')
514
- }
515
- else
516
- fluentd_spawn_cmd = ServerEngine.ruby_bin_path + " -Eascii-8bit:ascii-8bit "
517
- fluentd_spawn_cmd << ' ' + rubyopt + ' ' if rubyopt
518
- fluentd_spawn_cmd << $0.shellescape + ' '
519
- $fluentdargv.each{|a|
520
- fluentd_spawn_cmd << (a.shellescape + " ")
521
- }
522
- end
532
+ fluentd_spawn_cmd = [ServerEngine.ruby_bin_path, "-Eascii-8bit:ascii-8bit"]
533
+ fluentd_spawn_cmd << rubyopt if rubyopt
534
+ fluentd_spawn_cmd << $0
535
+ fluentd_spawn_cmd += $fluentdargv
536
+ fluentd_spawn_cmd << "--under-supervisor"
523
537
 
524
- fluentd_spawn_cmd << ("--under-supervisor")
525
- $log.info "spawn command to main: " + fluentd_spawn_cmd
538
+ $log.info "spawn command to main: ", cmdline: fluentd_spawn_cmd
526
539
 
527
540
  params = {}
528
541
  params['main_cmd'] = fluentd_spawn_cmd
@@ -617,39 +630,54 @@ module Fluent
617
630
  }.run
618
631
  end
619
632
 
633
+ def logging_with_console_output
634
+ yield $log
635
+ unless @log.stdout?
636
+ logger = ServerEngine::DaemonLogger.new(STDOUT)
637
+ log = Fluent::Log.new(logger)
638
+ log.level = @log_level
639
+ console = log.enable_debug
640
+ yield console
641
+ end
642
+ end
643
+
620
644
  def main_process(&block)
621
645
  Process.setproctitle("worker:#{@process_name}") if @process_name
622
646
 
623
- configuration_error = false
647
+ unrecoverable_error = false
624
648
 
625
649
  begin
626
650
  block.call
627
- rescue Fluent::ConfigError
628
- $log.error "config error", file: @config_path, error: $!.to_s
629
- $log.debug_backtrace
630
- unless @log.stdout?
631
- logger = ServerEngine::DaemonLogger.new(STDOUT)
632
- log = Fluent::Log.new(logger)
633
- log.level = @log_level
634
- console = log.enable_debug
635
- console.error "config error", file: @config_path, error: $!.to_s
636
- console.debug_backtrace
651
+ rescue Fluent::ConfigError => e
652
+ logging_with_console_output do |log|
653
+ log.error "config error", file: @config_path, error: e
654
+ log.debug_backtrace
655
+ end
656
+ unrecoverable_error = true
657
+ rescue Fluent::UnrecoverableError => e
658
+ logging_with_console_output do |log|
659
+ log.error e.message, error: e
660
+ log.error_backtrace
661
+ end
662
+ unrecoverable_error = true
663
+ rescue ScriptError => e # LoadError, NotImplementedError, SyntaxError
664
+ logging_with_console_output do |log|
665
+ if e.respond_to?(:path)
666
+ log.error e.message, path: e.path, error: e
667
+ else
668
+ log.error e.message, error: e
669
+ end
670
+ log.error_backtrace
637
671
  end
638
- configuration_error = true
639
- rescue
640
- $log.error "unexpected error", error: $!.to_s
641
- $log.error_backtrace
642
- unless @log.stdout?
643
- logger = ServerEngine::DaemonLogger.new(STDOUT)
644
- log = Fluent::Log.new(logger)
645
- log.level = @log_level
646
- console = log.enable_debug
647
- console.error "unexpected error", error: $!.to_s
648
- console.error_backtrace
672
+ unrecoverable_error = true
673
+ rescue => e
674
+ logging_with_console_output do |log|
675
+ log.error "unexpected error", error: e
676
+ log.error_backtrace
649
677
  end
650
678
  end
651
679
 
652
- exit!(configuration_error ? 2 : 1)
680
+ exit!(unrecoverable_error ? 2 : 1)
653
681
  end
654
682
 
655
683
  def read_config
@@ -21,6 +21,14 @@ module Fluent
21
21
  class SystemConfig
22
22
  include Configurable
23
23
 
24
+ SYSTEM_CONFIG_PARAMETERS = [
25
+ :root_dir, :log_level,
26
+ :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
27
+ :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
28
+ :file_permission, :dir_permission,
29
+ ]
30
+
31
+ config_param :root_dir, :string, default: nil
24
32
  config_param :log_level, default: nil do |level|
25
33
  Log.str_to_level(level)
26
34
  end
@@ -68,33 +76,28 @@ module Fluent
68
76
 
69
77
  def dup
70
78
  s = SystemConfig.new
71
- s.log_level = @log_level
72
- s.suppress_repeated_stacktrace = @suppress_repeated_stacktrace
73
- s.emit_error_log_interval = @emit_error_log_interval
74
- s.suppress_config_dump = @suppress_config_dump
75
- s.without_source = @without_source
76
- s.rpc_endpoint = @rpc_endpoint
77
- s.enable_get_dump = @enable_get_dump
78
- s.process_name = @process_name
79
- s.file_permission = @file_permission
80
- s.dir_permission = @dir_permission
81
-
79
+ SYSTEM_CONFIG_PARAMETERS.each do |param|
80
+ s.__send__("#{param}=", instance_variable_get("@#{param}"))
81
+ end
82
82
  s
83
83
  end
84
84
 
85
85
  def apply(supervisor)
86
86
  system = self
87
87
  supervisor.instance_eval {
88
- @log.level = @log_level = system.log_level unless system.log_level.nil?
89
- @suppress_interval = system.emit_error_log_interval unless system.emit_error_log_interval.nil?
90
- @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
91
- @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
92
- @without_source = system.without_source unless system.without_source.nil?
93
- @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
94
- @enable_get_dump = system.enable_get_dump unless system.enable_get_dump.nil?
95
- @process_name = system.process_name unless system.process_name.nil?
96
- @file_permission = system.file_permission unless system.file_permission.nil?
97
- @dir_permission = system.dir_permission unless system.dir_permission.nil?
88
+ SYSTEM_CONFIG_PARAMETERS.each do |param|
89
+ param_value = system.send(param)
90
+ next if param_value.nil?
91
+
92
+ case param
93
+ when :log_level
94
+ @log.level = @log_level = param_value
95
+ when :emit_error_log_interval
96
+ @suppress_interval = param_value
97
+ else
98
+ instance_variable_set("@#{param}", param_value)
99
+ end
100
+ end
98
101
  }
99
102
  end
100
103
 
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '0.14.10'
19
+ VERSION = '0.14.11'
20
20
 
21
21
  end
@@ -0,0 +1,348 @@
1
+ require_relative '../helper'
2
+
3
+ # require 'fluent/command/fluentd'
4
+ # don't require it... it runs immediately
5
+
6
+ require 'fileutils'
7
+ require 'timeout'
8
+
9
+ class TestFluentdCommand < ::Test::Unit::TestCase
10
+ TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluentd#{ENV['TEST_ENV_NUMBER']}")
11
+ SUPERVISOR_PID_PATTERN = /starting fluentd-[.0-9]+ pid=(\d+)/
12
+ WORKER_PID_PATTERN = /starting fluentd worker pid=(\d+) /
13
+
14
+ setup do
15
+ FileUtils.rm_rf(TMP_DIR)
16
+ FileUtils.mkdir_p(TMP_DIR)
17
+ @supervisor_pid = nil
18
+ @worker_pids = []
19
+ end
20
+
21
+ def process_exist?(pid)
22
+ begin
23
+ r = Process.waitpid(pid, Process::WNOHANG)
24
+ return true if r.nil?
25
+ false
26
+ rescue SystemCallError
27
+ false
28
+ end
29
+ end
30
+
31
+ def create_conf_file(name, content)
32
+ conf_path = File.join(TMP_DIR, name)
33
+ File.open(conf_path, 'w') do |file|
34
+ file.write content
35
+ end
36
+ conf_path
37
+ end
38
+
39
+ def create_plugin_file(name, content)
40
+ file_path = File.join(TMP_DIR, 'plugin', name)
41
+ FileUtils.mkdir_p(File.dirname(file_path))
42
+ File.open(file_path, 'w') do |file|
43
+ file.write content
44
+ end
45
+ file_path
46
+ end
47
+
48
+ def create_cmdline(conf_path, *fluentd_options)
49
+ cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
50
+ ["bundle", "exec", "ruby", cmd_path, "-c", conf_path, *fluentd_options]
51
+ end
52
+
53
+ def execute_command(cmdline, chdir=TMP_DIR)
54
+ null_stream = File.open(File::NULL, 'w')
55
+ gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
56
+
57
+ env = {
58
+ "BUNDLE_GEMFILE" => gemfile_path,
59
+ }
60
+ cmdname = cmdline.shift
61
+ arg0 = "testing-fluentd"
62
+ # p(here: "executing process", env: env, cmdname: cmdname, arg0: arg0, args: cmdline)
63
+ IO.popen(env, [[cmdname, arg0], *cmdline], chdir: chdir, err: [:child, :out]) do |io|
64
+ pid = io.pid
65
+ begin
66
+ yield pid, io
67
+ # p(here: "execute command", pid: pid, worker_pids: @worker_pids)
68
+ ensure
69
+ Process.kill(:KILL, pid) rescue nil
70
+ if @supervisor_pid
71
+ Process.kill(:KILL, @supervisor_pid) rescue nil
72
+ end
73
+ @worker_pids.each do |cpid|
74
+ Process.kill(:KILL, cpid) rescue nil
75
+ end
76
+ # p(here: "execute command", pid: pid, exist: process_exist?(pid), worker_pids: @worker_pids, exists: @worker_pids.map{|i| process_exist?(i) })
77
+ Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
78
+ end
79
+ end
80
+ ensure
81
+ null_stream.close rescue nil
82
+ end
83
+
84
+ def assert_log_matches(cmdline, *pattern_list, timeout: 10)
85
+ matched = false
86
+ assert_error_msg = "matched correctly"
87
+ stdio_buf = ""
88
+ begin
89
+ execute_command(cmdline) do |pid, stdout|
90
+ begin
91
+ waiting(timeout) do
92
+ while process_exist?(pid) && !matched
93
+ readables, _, _ = IO.select([stdout], nil, nil, 1)
94
+ next unless readables
95
+ break if readables.first.eof?
96
+
97
+ buf = readables.first.readpartial(1024)
98
+ # puts buf
99
+ stdio_buf << buf
100
+ lines = stdio_buf.split("\n")
101
+ if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
102
+ matched = true
103
+ end
104
+ end
105
+ end
106
+ ensure
107
+ if SUPERVISOR_PID_PATTERN =~ stdio_buf
108
+ @supervisor_pid = $1.to_i
109
+ end
110
+ stdio_buf.scan(WORKER_PID_PATTERN) do |worker_pid|
111
+ @worker_pids << worker_pid.first.to_i
112
+ end
113
+ end
114
+ end
115
+ rescue Timeout::Error
116
+ assert_error_msg = "execution timeout with command out:\n" + stdio_buf
117
+ rescue => e
118
+ assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
119
+ end
120
+ assert matched, assert_error_msg
121
+ end
122
+
123
+ def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
124
+ # empty_list.all?{ ... } is always true
125
+ matched = false
126
+ running = false
127
+ assert_error_msg = "failed to start correctly"
128
+ stdio_buf = ""
129
+ begin
130
+ execute_command(cmdline) do |pid, stdout|
131
+ begin
132
+ waiting(timeout) do
133
+ while process_exist?(pid) && !running
134
+ readables, _, _ = IO.select([stdout], nil, nil, 1)
135
+ next unless readables
136
+ next if readables.first.eof?
137
+
138
+ stdio_buf << readables.first.readpartial(1024)
139
+ lines = stdio_buf.split("\n")
140
+ if lines.any?{|line| line.include?("fluentd worker is now running") }
141
+ running = true
142
+ end
143
+ if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
144
+ matched = true
145
+ end
146
+ end
147
+ end
148
+ ensure
149
+ if SUPERVISOR_PID_PATTERN =~ stdio_buf
150
+ @supervisor_pid = $1.to_i
151
+ end
152
+ stdio_buf.scan(WORKER_PID_PATTERN) do |worker_pid|
153
+ @worker_pids << worker_pid.first.to_i
154
+ end
155
+ end
156
+ end
157
+ rescue Timeout::Error
158
+ assert_error_msg = "execution timeout with command out:\n" + stdio_buf
159
+ rescue => e
160
+ assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
161
+ assert false, assert_error_msg
162
+ end
163
+ assert !running, "fluentd started to run incorrectly:\n" + stdio_buf
164
+ unless matched
165
+ assert_error_msg = "fluentd failed to start, without specified regular expressions:\n" + stdio_buf
166
+ end
167
+ assert matched, assert_error_msg
168
+ end
169
+
170
+ sub_test_case 'with valid configuration' do
171
+ test 'runs successfully' do
172
+ conf = <<CONF
173
+ <source>
174
+ @type dummy
175
+ @id dummy
176
+ @label @dummydata
177
+ tag dummy
178
+ dummy {"message": "yay!"}
179
+ </source>
180
+ <label @dummydata>
181
+ <match dummy>
182
+ @type null
183
+ @id blackhole
184
+ </match>
185
+ </label>
186
+ CONF
187
+ conf_path = create_conf_file('valid.conf', conf)
188
+ assert File.exist?(conf_path)
189
+
190
+ assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker="0"')
191
+ end
192
+ end
193
+
194
+ sub_test_case 'with system configuration about root directory' do
195
+ setup do
196
+ @root_path = File.join(TMP_DIR, "rootpath")
197
+ FileUtils.rm_rf(@root_path)
198
+ @conf = <<CONF
199
+ <system>
200
+ root_dir #{@root_path}
201
+ </system>
202
+ <source>
203
+ @type dummy
204
+ @id dummy
205
+ @label @dummydata
206
+ tag dummy
207
+ dummy {"message": "yay!"}
208
+ </source>
209
+ <label @dummydata>
210
+ <match dummy>
211
+ @type null
212
+ @id blackhole
213
+ </match>
214
+ </label>
215
+ CONF
216
+ end
217
+
218
+ test 'use the specified existing directory as root' do
219
+ FileUtils.mkdir_p(@root_path)
220
+ conf_path = create_conf_file('existing_root_dir.conf', @conf)
221
+ assert Dir.exist?(@root_path)
222
+
223
+ assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker="0"')
224
+ end
225
+
226
+ test 'creates the specified root directory if missing' do
227
+ conf_path = create_conf_file('missing_root_dir.conf', @conf)
228
+ assert_false Dir.exist?(@root_path)
229
+
230
+ assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker="0"')
231
+ assert Dir.exist?(@root_path)
232
+ end
233
+
234
+ test 'fails to launch fluentd if specified root path is invalid path for directory' do
235
+ File.open(@root_path, 'w') do |_|
236
+ # create file and close it
237
+ end
238
+ conf_path = create_conf_file('existing_root_dir.conf', @conf)
239
+
240
+ assert_fluentd_fails_to_start(
241
+ create_cmdline(conf_path),
242
+ "non directory entry exists:#{@root_path} (Fluent::InvalidRootDirectory)",
243
+ )
244
+ end
245
+ end
246
+
247
+ sub_test_case 'configuration with wrong plugin type' do
248
+ test 'failed to start' do
249
+ conf = <<CONF
250
+ <source>
251
+ @type
252
+ @id dummy
253
+ @label @dummydata
254
+ tag dummy
255
+ dummy {"message": "yay!"}
256
+ </source>
257
+ <label @dummydata>
258
+ <match dummy>
259
+ @type null
260
+ @id blackhole
261
+ </match>
262
+ </label>
263
+ CONF
264
+ conf_path = create_conf_file('type_missing.conf', conf)
265
+ assert File.exist?(conf_path)
266
+
267
+ assert_fluentd_fails_to_start(
268
+ create_cmdline(conf_path),
269
+ "config error",
270
+ "error=\"Unknown input plugin ''. Run 'gem search -rd fluent-plugin' to find plugins",
271
+ )
272
+ end
273
+ end
274
+
275
+ sub_test_case 'configuration to load plugin file with syntax error' do
276
+ test 'failed to start' do
277
+ script = "require 'fluent/plugin/input'\n"
278
+ script << "module Fluent::Plugin\n"
279
+ script << " class BuggyInput < Input\n"
280
+ script << " Fluent::Plugin.register_input('buggy', self)\n"
281
+ script << " end\n"
282
+ plugin_path = create_plugin_file('in_buggy.rb', script)
283
+
284
+ conf = <<CONF
285
+ <source>
286
+ @type buggy
287
+ @id dummy
288
+ @label @dummydata
289
+ tag dummy
290
+ dummy {"message": "yay!"}
291
+ </source>
292
+ <label @dummydata>
293
+ <match dummy>
294
+ @type null
295
+ @id blackhole
296
+ </match>
297
+ </label>
298
+ CONF
299
+ conf_path = create_conf_file('buggy_plugin.conf', conf)
300
+ assert File.exist?(conf_path)
301
+
302
+ assert_fluentd_fails_to_start(
303
+ create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
304
+ "error_class=SyntaxError",
305
+ "in_buggy.rb:5: syntax error, unexpected end-of-input, expecting keyword_end",
306
+ )
307
+ end
308
+ end
309
+
310
+ sub_test_case 'configuration to load plugin which raises unrecoverable error in #start' do
311
+ test 'failed to start' do
312
+ script = "require 'fluent/plugin/input'\n"
313
+ script << "require 'fluent/error'\n"
314
+ script << "module Fluent::Plugin\n"
315
+ script << " class CrashingInput < Input\n"
316
+ script << " Fluent::Plugin.register_input('crashing', self)\n"
317
+ script << " def start\n"
318
+ script << " raise Fluent::UnrecoverableError"
319
+ script << " end\n"
320
+ script << " end\n"
321
+ script << "end\n"
322
+ plugin_path = create_plugin_file('in_crashing.rb', script)
323
+
324
+ conf = <<CONF
325
+ <source>
326
+ @type crashing
327
+ @id dummy
328
+ @label @dummydata
329
+ tag dummy
330
+ dummy {"message": "yay!"}
331
+ </source>
332
+ <label @dummydata>
333
+ <match dummy>
334
+ @type null
335
+ @id blackhole
336
+ </match>
337
+ </label>
338
+ CONF
339
+ conf_path = create_conf_file('crashing_plugin.conf', conf)
340
+ assert File.exist?(conf_path)
341
+
342
+ assert_fluentd_fails_to_start(
343
+ create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
344
+ 'unexpected error error_class=Fluent::UnrecoverableError error="an unrecoverable error occurs in Fluentd process"',
345
+ )
346
+ end
347
+ end
348
+ end