fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +2 -0
  4. data/CONTRIBUTING.md +6 -1
  5. data/ChangeLog +95 -0
  6. data/Rakefile +21 -0
  7. data/appveyor.yml +1 -0
  8. data/code-of-conduct.md +3 -0
  9. data/example/out_exec_filter.conf +42 -0
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/agent.rb +2 -2
  12. data/lib/fluent/command/binlog_reader.rb +1 -1
  13. data/lib/fluent/command/cat.rb +15 -4
  14. data/lib/fluent/compat/output.rb +14 -9
  15. data/lib/fluent/compat/parser.rb +141 -11
  16. data/lib/fluent/config/configure_proxy.rb +2 -11
  17. data/lib/fluent/config/section.rb +8 -1
  18. data/lib/fluent/configurable.rb +1 -3
  19. data/lib/fluent/env.rb +1 -1
  20. data/lib/fluent/log.rb +1 -1
  21. data/lib/fluent/plugin/base.rb +17 -0
  22. data/lib/fluent/plugin/filter_parser.rb +108 -0
  23. data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
  24. data/lib/fluent/plugin/filter_stdout.rb +1 -1
  25. data/lib/fluent/plugin/formatter.rb +5 -0
  26. data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
  27. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  28. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  29. data/lib/fluent/plugin/in_exec.rb +48 -93
  30. data/lib/fluent/plugin/in_forward.rb +66 -265
  31. data/lib/fluent/plugin/in_http.rb +68 -65
  32. data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
  33. data/lib/fluent/plugin/in_syslog.rb +42 -58
  34. data/lib/fluent/plugin/in_tail.rb +29 -14
  35. data/lib/fluent/plugin/in_tcp.rb +54 -14
  36. data/lib/fluent/plugin/in_udp.rb +49 -13
  37. data/lib/fluent/plugin/multi_output.rb +1 -3
  38. data/lib/fluent/plugin/out_exec.rb +58 -71
  39. data/lib/fluent/plugin/out_exec_filter.rb +199 -279
  40. data/lib/fluent/plugin/out_file.rb +172 -81
  41. data/lib/fluent/plugin/out_forward.rb +229 -206
  42. data/lib/fluent/plugin/out_stdout.rb +6 -21
  43. data/lib/fluent/plugin/output.rb +90 -59
  44. data/lib/fluent/plugin/parser.rb +121 -61
  45. data/lib/fluent/plugin/parser_csv.rb +9 -3
  46. data/lib/fluent/plugin/parser_json.rb +37 -35
  47. data/lib/fluent/plugin/parser_ltsv.rb +11 -19
  48. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  49. data/lib/fluent/plugin/parser_regexp.rb +15 -42
  50. data/lib/fluent/plugin/parser_tsv.rb +8 -3
  51. data/lib/fluent/plugin_helper.rb +10 -1
  52. data/lib/fluent/plugin_helper/child_process.rb +139 -73
  53. data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
  54. data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
  55. data/lib/fluent/plugin_helper/event_loop.rb +24 -6
  56. data/lib/fluent/plugin_helper/extract.rb +16 -4
  57. data/lib/fluent/plugin_helper/formatter.rb +9 -11
  58. data/lib/fluent/plugin_helper/inject.rb +16 -1
  59. data/lib/fluent/plugin_helper/parser.rb +3 -3
  60. data/lib/fluent/plugin_helper/server.rb +494 -0
  61. data/lib/fluent/plugin_helper/socket.rb +101 -0
  62. data/lib/fluent/plugin_helper/socket_option.rb +84 -0
  63. data/lib/fluent/plugin_helper/timer.rb +1 -0
  64. data/lib/fluent/root_agent.rb +1 -1
  65. data/lib/fluent/test/driver/base.rb +95 -49
  66. data/lib/fluent/test/driver/base_owner.rb +18 -8
  67. data/lib/fluent/test/driver/multi_output.rb +2 -1
  68. data/lib/fluent/test/driver/output.rb +29 -6
  69. data/lib/fluent/test/helpers.rb +3 -1
  70. data/lib/fluent/test/log.rb +4 -0
  71. data/lib/fluent/test/startup_shutdown.rb +13 -0
  72. data/lib/fluent/time.rb +14 -8
  73. data/lib/fluent/version.rb +1 -1
  74. data/lib/fluent/winsvc.rb +1 -1
  75. data/test/command/test_binlog_reader.rb +5 -1
  76. data/test/compat/test_parser.rb +10 -0
  77. data/test/config/test_configurable.rb +193 -0
  78. data/test/config/test_configure_proxy.rb +0 -43
  79. data/test/helper.rb +36 -1
  80. data/test/plugin/test_base.rb +16 -0
  81. data/test/plugin/test_filter_parser.rb +665 -0
  82. data/test/plugin/test_filter_record_transformer.rb +36 -100
  83. data/test/plugin/test_filter_stdout.rb +18 -27
  84. data/test/plugin/test_in_dummy.rb +1 -1
  85. data/test/plugin/test_in_exec.rb +206 -94
  86. data/test/plugin/test_in_forward.rb +268 -347
  87. data/test/plugin/test_in_http.rb +310 -186
  88. data/test/plugin/test_in_monitor_agent.rb +65 -35
  89. data/test/plugin/test_in_syslog.rb +39 -3
  90. data/test/plugin/test_in_tcp.rb +78 -62
  91. data/test/plugin/test_in_udp.rb +101 -80
  92. data/test/plugin/test_out_exec.rb +223 -68
  93. data/test/plugin/test_out_exec_filter.rb +520 -169
  94. data/test/plugin/test_out_file.rb +637 -177
  95. data/test/plugin/test_out_forward.rb +242 -234
  96. data/test/plugin/test_out_null.rb +1 -1
  97. data/test/plugin/test_out_secondary_file.rb +4 -2
  98. data/test/plugin/test_out_stdout.rb +14 -35
  99. data/test/plugin/test_output_as_buffered.rb +60 -2
  100. data/test/plugin/test_parser.rb +359 -0
  101. data/test/plugin/test_parser_csv.rb +1 -2
  102. data/test/plugin/test_parser_json.rb +3 -4
  103. data/test/plugin/test_parser_labeled_tsv.rb +1 -2
  104. data/test/plugin/test_parser_none.rb +1 -2
  105. data/test/plugin/test_parser_regexp.rb +8 -4
  106. data/test/plugin/test_parser_tsv.rb +4 -3
  107. data/test/plugin_helper/test_child_process.rb +184 -0
  108. data/test/plugin_helper/test_compat_parameters.rb +88 -1
  109. data/test/plugin_helper/test_extract.rb +0 -1
  110. data/test/plugin_helper/test_formatter.rb +5 -2
  111. data/test/plugin_helper/test_inject.rb +21 -0
  112. data/test/plugin_helper/test_parser.rb +6 -5
  113. data/test/plugin_helper/test_server.rb +905 -0
  114. data/test/test_event_time.rb +3 -1
  115. data/test/test_output.rb +53 -2
  116. data/test/test_plugin_classes.rb +20 -0
  117. data/test/test_root_agent.rb +139 -0
  118. data/test/test_test_drivers.rb +135 -0
  119. metadata +28 -8
  120. data/test/plugin/test_parser_base.rb +0 -32
@@ -20,13 +20,19 @@ require 'csv'
20
20
 
21
21
  module Fluent
22
22
  module Plugin
23
- class CSVParser < ValuesParser
23
+ class CSVParser < Parser
24
24
  Plugin.register_parser('csv', self)
25
25
 
26
+ desc 'Names of fields included in each lines'
27
+ config_param :keys, :array, value_type: :string
28
+ desc 'The delimiter character (or string) of CSV values'
26
29
  config_param :delimiter, :string, default: ','
27
30
 
28
- def parse(text)
29
- yield values_map(CSV.parse_line(text, col_sep: @delimiter))
31
+ def parse(text, &block)
32
+ values = CSV.parse_line(text, col_sep: @delimiter)
33
+ r = Hash[@keys.zip(values)]
34
+ time, record = convert_values(parse_time(r), r)
35
+ yield time, record
30
36
  end
31
37
  end
32
38
  end
@@ -19,61 +19,63 @@ require 'fluent/env'
19
19
  require 'fluent/time'
20
20
 
21
21
  require 'yajl'
22
+ require 'json'
22
23
 
23
24
  module Fluent
24
25
  module Plugin
25
26
  class JSONParser < Parser
26
27
  Plugin.register_parser('json', self)
27
28
 
28
- config_param :time_key, :string, default: 'time'
29
- config_param :json_parser, :string, default: 'oj'
29
+ config_set_default :time_key, 'time'
30
+ config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
30
31
 
31
- def configure(conf)
32
- super
32
+ config_set_default :time_type, :float
33
33
 
34
- if @time_format
35
- @time_parser = time_parser_create
36
- @mutex = Mutex.new
34
+ def configure(conf)
35
+ if conf.has_key?('time_format')
36
+ conf['time_type'] ||= 'string'
37
37
  end
38
38
 
39
- begin
40
- raise LoadError unless @json_parser == 'oj'
39
+ super
40
+ @load_proc, @error_class = configure_json_parser(@json_parser)
41
+ end
42
+
43
+ def configure_json_parser(name)
44
+ case name
45
+ when :oj
41
46
  require 'oj'
42
47
  Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
43
- @load_proc = Oj.method(:load)
44
- @error_class = Oj::ParseError
45
- rescue LoadError
46
- @load_proc = Yajl.method(:load)
47
- @error_class = Yajl::ParseError
48
+ [Oj.method(:load), Oj::ParseError]
49
+ when :json then [JSON.method(:load), JSON::ParserError]
50
+ when :yajl then [Yajl.method(:load), Yajl::ParseError]
51
+ else
52
+ raise "BUG: unknown json parser specified: #{name}"
48
53
  end
54
+ rescue LoadError
55
+ name = :yajl
56
+ log.info "Oj is not installed, and failing back to Yajl for json parser" if log
57
+ retry
49
58
  end
50
59
 
51
60
  def parse(text)
52
- record = @load_proc.call(text)
53
-
54
- value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
55
- if value
56
- if @time_format
57
- time = @mutex.synchronize { @time_parser.parse(value) }
58
- else
59
- begin
60
- time = Fluent::EventTime.from_time(Time.at(value.to_f))
61
- rescue => e
62
- raise ParserError, "invalid time value: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
63
- end
64
- end
65
- else
66
- if @estimate_current_event
67
- time = Fluent::EventTime.now
68
- else
69
- time = nil
70
- end
71
- end
72
-
61
+ r = @load_proc.call(text)
62
+ time, record = convert_values(parse_time(r), r)
73
63
  yield time, record
74
64
  rescue @error_class
75
65
  yield nil, nil
76
66
  end
67
+
68
+ def parser_type
69
+ :text
70
+ end
71
+
72
+ def parse_io(io, &block)
73
+ y = Yajl::Parser.new
74
+ y.on_parse_complete = ->(record){
75
+ block.call(parse_time(record), record)
76
+ }
77
+ y.parse(io)
78
+ end
77
79
  end
78
80
  end
79
81
  end
@@ -15,35 +15,27 @@
15
15
  #
16
16
 
17
17
  require 'fluent/plugin/parser'
18
- require 'fluent/time'
19
18
 
20
19
  module Fluent
21
20
  module Plugin
22
- class LabeledTSVParser < ValuesParser
21
+ class LabeledTSVParser < Parser
23
22
  Plugin.register_parser('ltsv', self)
24
23
 
25
- config_param :delimiter, :string, default: "\t"
24
+ desc 'The delimiter character (or string) of TSV values'
25
+ config_param :delimiter, :string, default: "\t"
26
+ desc 'The delimiter character between field name and value'
26
27
  config_param :label_delimiter, :string, default: ":"
27
- config_param :time_key, :string, default: "time"
28
28
 
29
- def configure(conf)
30
- # this assignment is not to raise ConfigError in ValuesParser#configure
31
- conf['keys'] = conf['time_key'] || 'time'
32
- super(conf)
33
- end
29
+ config_set_default :time_key, 'time'
34
30
 
35
31
  def parse(text)
36
- # TODO: thread unsafe: @keys might be changed by other threads
37
- @keys = []
38
- values = []
39
-
40
- text.split(delimiter).each do |pair|
41
- key, value = pair.split(label_delimiter, 2)
42
- @keys.push(key)
43
- values.push(value)
32
+ r = {}
33
+ text.split(@delimiter).each do |pair|
34
+ key, value = pair.split(@label_delimiter, 2)
35
+ r[key] = value
44
36
  end
45
-
46
- yield values_map(values)
37
+ time, record = convert_values(parse_time(r), r)
38
+ yield time, record
47
39
  end
48
40
  end
49
41
  end
@@ -0,0 +1,50 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin/parser'
18
+ require 'fluent/msgpack_factory'
19
+
20
+ module Fluent
21
+ module Plugin
22
+ class MessagePackParser < Parser
23
+ Plugin.register_parser('msgpack', self)
24
+
25
+ def configure(conf)
26
+ super
27
+ @unpacker = Fluent::MessagePackFactory.engine_factory.unpacker
28
+ end
29
+
30
+ def parser_type
31
+ :binary
32
+ end
33
+
34
+ def parse(data)
35
+ @unpacker.feed_each(data) do |obj|
36
+ yield convert_values(parse_time(obj), obj)
37
+ end
38
+ end
39
+ alias parse_partial_data parse
40
+
41
+ def parse_io(io, &block)
42
+ u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
43
+ u.each do |obj|
44
+ time, record = convert_values(parse_time(obj), obj)
45
+ yield time, record
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -1,33 +1,26 @@
1
1
  module Fluent
2
2
  module Plugin
3
3
  class RegexpParser < Parser
4
- include Fluent::Compat::TypeConverter
5
-
6
4
  Plugin.register_parser("regexp", self)
7
5
 
8
- config_param :expression, :string, default: ""
6
+ config_param :expression, :string
9
7
  config_param :ignorecase, :bool, default: false
10
8
  config_param :multiline, :bool, default: false
11
- config_param :time_key, :string, default: 'time'
12
9
 
13
- def initialize
14
- super
15
- @mutex = Mutex.new
16
- end
10
+ config_set_default :time_key, 'time'
17
11
 
18
12
  def configure(conf)
19
13
  super
20
- @time_parser = time_parser_create
21
- unless @expression.empty?
22
- if @expression[0] == "/" && @expression[-1] == "/"
23
- regexp_option = 0
24
- regexp_option |= Regexp::IGNORECASE if @ignorecase
25
- regexp_option |= Regexp::MULTILINE if @multiline
26
- @regexp = Regexp.new(@expression[1..-2], regexp_option)
27
- else
28
- raise Fluent::ConfigError, "expression must start with `/` and end with `/`: #{@expression}"
29
- end
30
- end
14
+
15
+ expr = if @expression[0] == "/" && @expression[-1] == "/"
16
+ @expression[1..-2]
17
+ else
18
+ @expression
19
+ end
20
+ regexp_option = 0
21
+ regexp_option |= Regexp::IGNORECASE if @ignorecase
22
+ regexp_option |= Regexp::MULTILINE if @multiline
23
+ @regexp = Regexp.new(expr, regexp_option)
31
24
  end
32
25
 
33
26
  def parse(text)
@@ -37,34 +30,14 @@ module Fluent
37
30
  return
38
31
  end
39
32
 
40
- time = nil
41
- record = {}
42
-
33
+ r = {}
43
34
  m.names.each do |name|
44
35
  if value = m[name]
45
- if name == @time_key
46
- time = @mutex.synchronize { @time_parser.parse(value) }
47
- if @keep_time_key
48
- record[name] = if @type_converters.nil?
49
- value
50
- else
51
- convert_type(name, value)
52
- end
53
- end
54
- else
55
- record[name] = if @type_converters.nil?
56
- value
57
- else
58
- convert_type(name, value)
59
- end
60
- end
36
+ r[name] = value
61
37
  end
62
38
  end
63
39
 
64
- if @estimate_current_event
65
- time ||= Fluent::EventTime.now
66
- end
67
-
40
+ time, record = convert_values(parse_time(r), r)
68
41
  yield time, record
69
42
  end
70
43
  end
@@ -15,13 +15,15 @@
15
15
  #
16
16
 
17
17
  require 'fluent/plugin/parser'
18
- require 'fluent/time'
19
18
 
20
19
  module Fluent
21
20
  module Plugin
22
- class TSVParser < ValuesParser
21
+ class TSVParser < Parser
23
22
  Plugin.register_parser('tsv', self)
24
23
 
24
+ desc 'Names of fields included in each lines'
25
+ config_param :keys, :array, value_type: :string
26
+ desc 'The delimiter character (or string) of TSV values'
25
27
  config_param :delimiter, :string, default: "\t"
26
28
 
27
29
  def configure(conf)
@@ -30,7 +32,10 @@ module Fluent
30
32
  end
31
33
 
32
34
  def parse(text)
33
- yield values_map(text.split(@delimiter, @key_num))
35
+ values = text.split(@delimiter, @key_num)
36
+ r = Hash[@keys.zip(values)]
37
+ time, record = convert_values(parse_time(r), r)
38
+ yield time, record
34
39
  end
35
40
  end
36
41
  end
@@ -24,6 +24,8 @@ require 'fluent/plugin_helper/parser'
24
24
  require 'fluent/plugin_helper/formatter'
25
25
  require 'fluent/plugin_helper/inject'
26
26
  require 'fluent/plugin_helper/extract'
27
+ require 'fluent/plugin_helper/socket'
28
+ require 'fluent/plugin_helper/server'
27
29
  require 'fluent/plugin_helper/retry_state'
28
30
  require 'fluent/plugin_helper/compat_parameters'
29
31
 
@@ -36,7 +38,14 @@ module Fluent
36
38
  end
37
39
 
38
40
  def helpers(*snake_case_symbols)
39
- helper_modules = snake_case_symbols.map{|name| Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join) }
41
+ helper_modules = []
42
+ snake_case_symbols.each do |name|
43
+ begin
44
+ helper_modules << Fluent::PluginHelper.const_get(name.to_s.split('_').map(&:capitalize).join)
45
+ rescue NameError
46
+ raise "Unknown plugin helper:#{name}"
47
+ end
48
+ end
40
49
  include(*helper_modules)
41
50
  end
42
51
  end
@@ -49,39 +49,60 @@ module Fluent
49
49
  ::Thread.current[:_fluentd_plugin_helper_child_process_pid]
50
50
  end
51
51
 
52
- def child_process_exit_status
53
- ::Thread.current[:_fluentd_plugin_helper_child_process_exit_status]
52
+ def child_process_exist?(pid)
53
+ pinfo = @_child_process_processes[pid]
54
+ return false unless pinfo
55
+
56
+ return false if pinfo.exit_status
57
+
58
+ true
54
59
  end
55
60
 
61
+ # on_exit_callback = ->(status){ ... }
62
+ # status is an instance of Process::Status
63
+ # On Windows, exitstatus=0 and termsig=nil even when child process was killed.
56
64
  def child_process_execute(
57
65
  title, command,
58
66
  arguments: nil, subprocess_name: nil, interval: nil, immediate: false, parallel: false,
59
67
  mode: [:read, :write], stderr: :discard, env: {}, unsetenv: false, chdir: nil,
60
68
  internal_encoding: 'utf-8', external_encoding: 'ascii-8bit', scrub: true, replace_string: nil,
69
+ wait_timeout: nil, on_exit_callback: nil,
61
70
  &block
62
71
  )
63
72
  raise ArgumentError, "BUG: title must be a symbol" unless title.is_a? Symbol
64
73
  raise ArgumentError, "BUG: arguments required if subprocess name is replaced" if subprocess_name && !arguments
65
74
 
75
+ mode ||= []
76
+ mode = [] unless block
66
77
  raise ArgumentError, "BUG: invalid mode specification" unless mode.all?{|m| MODE_PARAMS.include?(m) }
67
78
  raise ArgumentError, "BUG: read_with_stderr is exclusive with :read and :stderr" if mode.include?(:read_with_stderr) && (mode.include?(:read) || mode.include?(:stderr))
68
79
  raise ArgumentError, "BUG: invalid stderr handling specification" unless STDERR_OPTIONS.include?(stderr)
69
80
 
70
- raise ArgumentError, "BUG: block not specified which receive i/o object" unless block_given?
71
- raise ArgumentError, "BUG: number of block arguments are different from size of mode" unless block.arity == mode.size
81
+ raise ArgumentError, "BUG: number of block arguments are different from size of mode" if block && block.arity != mode.size
72
82
 
73
83
  running = false
74
84
  callback = ->(*args) {
75
85
  running = true
76
86
  begin
77
- block.call(*args)
87
+ block && block.call(*args)
78
88
  ensure
79
89
  running = false
80
90
  end
81
91
  }
82
92
 
93
+ retval = nil
94
+ execute_child_process = ->(){
95
+ child_process_execute_once(
96
+ title, command, arguments,
97
+ subprocess_name, mode, stderr, env, unsetenv, chdir,
98
+ internal_encoding, external_encoding, scrub, replace_string,
99
+ wait_timeout, on_exit_callback,
100
+ &callback
101
+ )
102
+ }
103
+
83
104
  if immediate || !interval
84
- child_process_execute_once(title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir, internal_encoding, external_encoding, scrub, replace_string, &callback)
105
+ retval = execute_child_process.call
85
106
  end
86
107
 
87
108
  if interval
@@ -89,10 +110,12 @@ module Fluent
89
110
  if !parallel && running
90
111
  log.warn "previous child process is still running. skipped.", title: title, command: command, arguments: arguments, interval: interval, parallel: parallel
91
112
  else
92
- child_process_execute_once(title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir, internal_encoding, external_encoding, scrub, replace_string, &callback)
113
+ execute_child_process.call
93
114
  end
94
115
  end
95
116
  end
117
+
118
+ retval # nil if interval
96
119
  end
97
120
 
98
121
  def initialize
@@ -101,11 +124,8 @@ module Fluent
101
124
  @_child_process_exit_timeout = CHILD_PROCESS_DEFAULT_EXIT_TIMEOUT
102
125
  @_child_process_kill_timeout = CHILD_PROCESS_DEFAULT_KILL_TIMEOUT
103
126
  @_child_process_mutex = Mutex.new
104
- end
105
-
106
- def start
107
- super
108
127
  @_child_process_processes = {} # pid => ProcessInfo
128
+ @_child_process_clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
109
129
  end
110
130
 
111
131
  def stop
@@ -115,86 +135,102 @@ module Fluent
115
135
  process_info.thread[:_fluentd_plugin_helper_child_process_running] = false
116
136
  end
117
137
  end
138
+
139
+ super
118
140
  end
119
141
 
120
142
  def shutdown
121
143
  @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
122
144
  process_info = @_child_process_processes[pid]
123
- next if !process_info || !process_info.writeio_in_use
124
- begin
125
- Timeout.timeout(@_child_process_exit_timeout) do
126
- process_info.writeio.close
127
- end
128
- rescue Timeout::Error
129
- log.debug "External process #{process_info.title} doesn't exist after STDIN close in timeout #{@_child_process_exit_timeout}sec"
130
- end
145
+ next if !process_info
146
+ process_info.writeio && process_info.writeio.close rescue nil
147
+ end
148
+
149
+ super
131
150
 
151
+ @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
152
+ process_info = @_child_process_processes[pid]
153
+ next if !process_info
132
154
  child_process_kill(process_info)
133
155
  end
134
156
 
135
- super
157
+ exit_wait_timeout = Process.clock_gettime(@_child_process_clock_id) + @_child_process_exit_timeout
158
+ while Process.clock_gettime(@_child_process_clock_id) < exit_wait_timeout
159
+ process_exists = false
160
+ @_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
161
+ unless @_child_process_processes[pid].exit_status
162
+ process_exists = true
163
+ break
164
+ end
165
+ end
166
+ if process_exists
167
+ sleep CHILD_PROCESS_LOOP_CHECK_INTERVAL
168
+ else
169
+ break
170
+ end
171
+ end
136
172
  end
137
173
 
138
174
  def close
139
- while (pids = @_child_process_mutex.synchronize{ @_child_process_processes.keys }).size > 0
175
+ while true
176
+ pids = @_child_process_mutex.synchronize{ @_child_process_processes.keys }
177
+ break if pids.size < 1
178
+
179
+ living_process_exist = false
140
180
  pids.each do |pid|
141
181
  process_info = @_child_process_processes[pid]
142
- if !process_info || !process_info.alive
143
- @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
144
- next
145
- end
182
+ next if !process_info || process_info.exit_status
146
183
 
147
- process_info.killed_at ||= Time.now # for illegular case (e.g., created after shutdown)
148
- next if Time.now < process_info.killed_at + @_child_process_kill_timeout
184
+ living_process_exist = true
185
+
186
+ process_info.killed_at ||= Process.clock_gettime(@_child_process_clock_id) # for illegular case (e.g., created after shutdown)
187
+ timeout_at = process_info.killed_at + @_child_process_kill_timeout
188
+ now = Process.clock_gettime(@_child_process_clock_id)
189
+ next if now < timeout_at
149
190
 
150
191
  child_process_kill(process_info, force: true)
151
- @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
152
192
  end
153
193
 
194
+ break if living_process_exist
195
+
154
196
  sleep CHILD_PROCESS_LOOP_CHECK_INTERVAL
155
197
  end
156
198
 
157
199
  super
158
200
  end
159
201
 
160
- def child_process_kill(process_info, force: false)
161
- if !process_info || !process_info.alive
162
- return
163
- end
202
+ def terminate
203
+ @_child_process_processes = {}
164
204
 
165
- process_info.killed_at = Time.now unless force
205
+ super
206
+ end
166
207
 
167
- begin
168
- pid, status = Process.waitpid2(process_info.pid, Process::WNOHANG)
169
- if pid && status
170
- process_info.thread[:_fluentd_plugin_helper_child_process_exit_status] = status
171
- process_info.alive = false
172
- end
173
- rescue Errno::ECHILD, Errno::ESRCH, Errno::EPERM
174
- process_info.alive = false
175
- rescue
176
- # ignore
177
- end
178
- if !process_info.alive
179
- return
180
- end
208
+ def child_process_kill(pinfo, force: false)
209
+ return if !pinfo
210
+ pinfo.killed_at = Process.clock_gettime(@_child_process_clock_id) unless force
181
211
 
212
+ pid = pinfo.pid
182
213
  begin
183
- signal = (Fluent.windows? || force) ? :KILL : :TERM
184
- Process.kill(signal, process_info.pid)
185
- if force
186
- process_info.alive = false
214
+ if !pinfo.exit_status && child_process_exist?(pid)
215
+ signal = (Fluent.windows? || force) ? :KILL : :TERM
216
+ Process.kill(signal, pinfo.pid)
187
217
  end
188
218
  rescue Errno::ECHILD, Errno::ESRCH
189
- process_info.alive = false
219
+ # ignore
190
220
  end
191
221
  end
192
222
 
193
- ProcessInfo = Struct.new(:title, :thread, :pid, :readio, :readio_in_use, :writeio, :writeio_in_use, :stderrio, :stderrio_in_use, :wait_thread, :alive, :killed_at)
223
+ ProcessInfo = Struct.new(
224
+ :title,
225
+ :thread, :pid,
226
+ :readio, :readio_in_use, :writeio, :writeio_in_use, :stderrio, :stderrio_in_use,
227
+ :wait_thread, :alive, :killed_at, :exit_status,
228
+ :on_exit_callback, :on_exit_callback_mutex,
229
+ )
194
230
 
195
231
  def child_process_execute_once(
196
232
  title, command, arguments, subprocess_name, mode, stderr, env, unsetenv, chdir,
197
- internal_encoding, external_encoding, scrub, replace_string, &block
233
+ internal_encoding, external_encoding, scrub, replace_string, wait_timeout, on_exit_callback, &block
198
234
  )
199
235
  spawn_args = if arguments || subprocess_name
200
236
  [ env, (subprocess_name ? [command, subprocess_name] : command), *(arguments || []) ]
@@ -227,36 +263,40 @@ module Fluent
227
263
  writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
228
264
  else
229
265
  writeio, readio, stderrio, wait_thread = *Open3.popen3(*spawn_args, spawn_opts)
230
- if !mode.include?(:stderr) # stderr == :discard
231
- stderrio.reopen(IO::NULL)
232
- end
233
266
  end
234
267
 
235
268
  if mode.include?(:write)
236
269
  writeio.set_encoding(external_encoding, internal_encoding, encoding_options)
237
270
  writeio_in_use = true
271
+ else
272
+ writeio.reopen(IO::NULL) if writeio
238
273
  end
239
274
  if mode.include?(:read) || mode.include?(:read_with_stderr)
240
275
  readio.set_encoding(external_encoding, internal_encoding, encoding_options)
241
276
  readio_in_use = true
277
+ else
278
+ readio.reopen(IO::NULL) if readio
242
279
  end
243
280
  if mode.include?(:stderr)
244
281
  stderrio.set_encoding(external_encoding, internal_encoding, encoding_options)
245
282
  stderrio_in_use = true
283
+ else
284
+ stderrio.reopen(IO::NULL) if stderrio && stderrio == :discard
246
285
  end
247
286
 
248
287
  pid = wait_thread.pid # wait_thread => Process::Waiter
249
288
 
250
289
  io_objects = []
251
290
  mode.each do |m|
252
- io_objects << case m
253
- when :read then readio
254
- when :write then writeio
255
- when :read_with_stderr then readio
256
- when :stderr then stderrio
257
- else
258
- raise "BUG: invalid mode must be checked before here: '#{m}'"
259
- end
291
+ io_obj = case m
292
+ when :read then readio
293
+ when :write then writeio
294
+ when :read_with_stderr then readio
295
+ when :stderr then stderrio
296
+ else
297
+ raise "BUG: invalid mode must be checked before here: '#{m}'"
298
+ end
299
+ io_objects << io_obj
260
300
  end
261
301
 
262
302
  m = Mutex.new
@@ -265,28 +305,54 @@ module Fluent
265
305
  m.lock # run after plugin thread get pid, thread instance and i/o
266
306
  m.unlock
267
307
  begin
268
- block.call(*io_objects)
308
+ @_child_process_processes[pid].alive = true
309
+ block.call(*io_objects) if block_given?
310
+ writeio.close if writeio
269
311
  rescue EOFError => e
270
312
  log.debug "Process exit and I/O closed", title: title, pid: pid, command: command, arguments: arguments
271
313
  rescue IOError => e
272
- if e.message == 'stream closed'
314
+ if e.message == 'stream closed' || e.message == 'closed stream' # "closed stream" is of ruby 2.1
273
315
  log.debug "Process I/O stream closed", title: title, pid: pid, command: command, arguments: arguments
274
316
  else
275
317
  log.error "Unexpected I/O error for child process", title: title, pid: pid, command: command, arguments: arguments, error: e
276
318
  end
319
+ rescue Errno::EPIPE => e
320
+ log.debug "Broken pipe, child process unexpectedly exits", title: title, pid: pid, command: command, arguments: arguments
277
321
  rescue => e
278
322
  log.warn "Unexpected error while processing I/O for child process", title: title, pid: pid, command: command, error: e
279
323
  end
280
- process_info = @_child_process_mutex.synchronize do
281
- process_info = @_child_process_processes[pid]
282
- @_child_process_processes.delete(pid)
283
- process_info
324
+
325
+ if wait_timeout
326
+ if wait_thread.join(wait_timeout) # Thread#join returns nil when limit expires
327
+ # wait_thread successfully exits
328
+ @_child_process_processes[pid].exit_status = wait_thread.value
329
+ else
330
+ log.warn "child process timed out", title: title, pid: pid, command: command, arguments: arguments
331
+ child_process_kill(@_child_process_processes[pid], force: true)
332
+ @_child_process_processes[pid].exit_status = wait_thread.value
333
+ end
334
+ else
335
+ @_child_process_processes[pid].exit_status = wait_thread.value # with join
336
+ end
337
+ process_info = @_child_process_mutex.synchronize{ @_child_process_processes.delete(pid) }
338
+
339
+ cb = process_info.on_exit_callback_mutex.synchronize do
340
+ cback = process_info.on_exit_callback
341
+ process_info.on_exit_callback = nil
342
+ cback
343
+ end
344
+ if cb
345
+ cb.call(process_info.exit_status) rescue nil
284
346
  end
285
- child_process_kill(process_info, force: true) if process_info && process_info.alive && ::Thread.current[:_fluentd_plugin_helper_child_process_running]
286
347
  end
287
348
  thread[:_fluentd_plugin_helper_child_process_running] = true
288
349
  thread[:_fluentd_plugin_helper_child_process_pid] = pid
289
- pinfo = ProcessInfo.new(title, thread, pid, readio, readio_in_use, writeio, writeio_in_use, stderrio, stderrio_in_use, wait_thread, true, nil)
350
+ pinfo = ProcessInfo.new(
351
+ title, thread, pid,
352
+ readio, readio_in_use, writeio, writeio_in_use, stderrio, stderrio_in_use,
353
+ wait_thread, false, nil, nil, on_exit_callback, Mutex.new
354
+ )
355
+
290
356
  @_child_process_mutex.synchronize do
291
357
  @_child_process_processes[pid] = pinfo
292
358
  end