fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +2 -0
  4. data/CONTRIBUTING.md +6 -1
  5. data/ChangeLog +95 -0
  6. data/Rakefile +21 -0
  7. data/appveyor.yml +1 -0
  8. data/code-of-conduct.md +3 -0
  9. data/example/out_exec_filter.conf +42 -0
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/agent.rb +2 -2
  12. data/lib/fluent/command/binlog_reader.rb +1 -1
  13. data/lib/fluent/command/cat.rb +15 -4
  14. data/lib/fluent/compat/output.rb +14 -9
  15. data/lib/fluent/compat/parser.rb +141 -11
  16. data/lib/fluent/config/configure_proxy.rb +2 -11
  17. data/lib/fluent/config/section.rb +8 -1
  18. data/lib/fluent/configurable.rb +1 -3
  19. data/lib/fluent/env.rb +1 -1
  20. data/lib/fluent/log.rb +1 -1
  21. data/lib/fluent/plugin/base.rb +17 -0
  22. data/lib/fluent/plugin/filter_parser.rb +108 -0
  23. data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
  24. data/lib/fluent/plugin/filter_stdout.rb +1 -1
  25. data/lib/fluent/plugin/formatter.rb +5 -0
  26. data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
  27. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  28. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  29. data/lib/fluent/plugin/in_exec.rb +48 -93
  30. data/lib/fluent/plugin/in_forward.rb +66 -265
  31. data/lib/fluent/plugin/in_http.rb +68 -65
  32. data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
  33. data/lib/fluent/plugin/in_syslog.rb +42 -58
  34. data/lib/fluent/plugin/in_tail.rb +29 -14
  35. data/lib/fluent/plugin/in_tcp.rb +54 -14
  36. data/lib/fluent/plugin/in_udp.rb +49 -13
  37. data/lib/fluent/plugin/multi_output.rb +1 -3
  38. data/lib/fluent/plugin/out_exec.rb +58 -71
  39. data/lib/fluent/plugin/out_exec_filter.rb +199 -279
  40. data/lib/fluent/plugin/out_file.rb +172 -81
  41. data/lib/fluent/plugin/out_forward.rb +229 -206
  42. data/lib/fluent/plugin/out_stdout.rb +6 -21
  43. data/lib/fluent/plugin/output.rb +90 -59
  44. data/lib/fluent/plugin/parser.rb +121 -61
  45. data/lib/fluent/plugin/parser_csv.rb +9 -3
  46. data/lib/fluent/plugin/parser_json.rb +37 -35
  47. data/lib/fluent/plugin/parser_ltsv.rb +11 -19
  48. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  49. data/lib/fluent/plugin/parser_regexp.rb +15 -42
  50. data/lib/fluent/plugin/parser_tsv.rb +8 -3
  51. data/lib/fluent/plugin_helper.rb +10 -1
  52. data/lib/fluent/plugin_helper/child_process.rb +139 -73
  53. data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
  54. data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
  55. data/lib/fluent/plugin_helper/event_loop.rb +24 -6
  56. data/lib/fluent/plugin_helper/extract.rb +16 -4
  57. data/lib/fluent/plugin_helper/formatter.rb +9 -11
  58. data/lib/fluent/plugin_helper/inject.rb +16 -1
  59. data/lib/fluent/plugin_helper/parser.rb +3 -3
  60. data/lib/fluent/plugin_helper/server.rb +494 -0
  61. data/lib/fluent/plugin_helper/socket.rb +101 -0
  62. data/lib/fluent/plugin_helper/socket_option.rb +84 -0
  63. data/lib/fluent/plugin_helper/timer.rb +1 -0
  64. data/lib/fluent/root_agent.rb +1 -1
  65. data/lib/fluent/test/driver/base.rb +95 -49
  66. data/lib/fluent/test/driver/base_owner.rb +18 -8
  67. data/lib/fluent/test/driver/multi_output.rb +2 -1
  68. data/lib/fluent/test/driver/output.rb +29 -6
  69. data/lib/fluent/test/helpers.rb +3 -1
  70. data/lib/fluent/test/log.rb +4 -0
  71. data/lib/fluent/test/startup_shutdown.rb +13 -0
  72. data/lib/fluent/time.rb +14 -8
  73. data/lib/fluent/version.rb +1 -1
  74. data/lib/fluent/winsvc.rb +1 -1
  75. data/test/command/test_binlog_reader.rb +5 -1
  76. data/test/compat/test_parser.rb +10 -0
  77. data/test/config/test_configurable.rb +193 -0
  78. data/test/config/test_configure_proxy.rb +0 -43
  79. data/test/helper.rb +36 -1
  80. data/test/plugin/test_base.rb +16 -0
  81. data/test/plugin/test_filter_parser.rb +665 -0
  82. data/test/plugin/test_filter_record_transformer.rb +36 -100
  83. data/test/plugin/test_filter_stdout.rb +18 -27
  84. data/test/plugin/test_in_dummy.rb +1 -1
  85. data/test/plugin/test_in_exec.rb +206 -94
  86. data/test/plugin/test_in_forward.rb +268 -347
  87. data/test/plugin/test_in_http.rb +310 -186
  88. data/test/plugin/test_in_monitor_agent.rb +65 -35
  89. data/test/plugin/test_in_syslog.rb +39 -3
  90. data/test/plugin/test_in_tcp.rb +78 -62
  91. data/test/plugin/test_in_udp.rb +101 -80
  92. data/test/plugin/test_out_exec.rb +223 -68
  93. data/test/plugin/test_out_exec_filter.rb +520 -169
  94. data/test/plugin/test_out_file.rb +637 -177
  95. data/test/plugin/test_out_forward.rb +242 -234
  96. data/test/plugin/test_out_null.rb +1 -1
  97. data/test/plugin/test_out_secondary_file.rb +4 -2
  98. data/test/plugin/test_out_stdout.rb +14 -35
  99. data/test/plugin/test_output_as_buffered.rb +60 -2
  100. data/test/plugin/test_parser.rb +359 -0
  101. data/test/plugin/test_parser_csv.rb +1 -2
  102. data/test/plugin/test_parser_json.rb +3 -4
  103. data/test/plugin/test_parser_labeled_tsv.rb +1 -2
  104. data/test/plugin/test_parser_none.rb +1 -2
  105. data/test/plugin/test_parser_regexp.rb +8 -4
  106. data/test/plugin/test_parser_tsv.rb +4 -3
  107. data/test/plugin_helper/test_child_process.rb +184 -0
  108. data/test/plugin_helper/test_compat_parameters.rb +88 -1
  109. data/test/plugin_helper/test_extract.rb +0 -1
  110. data/test/plugin_helper/test_formatter.rb +5 -2
  111. data/test/plugin_helper/test_inject.rb +21 -0
  112. data/test/plugin_helper/test_parser.rb +6 -5
  113. data/test/plugin_helper/test_server.rb +905 -0
  114. data/test/test_event_time.rb +3 -1
  115. data/test/test_output.rb +53 -2
  116. data/test/test_plugin_classes.rb +20 -0
  117. data/test/test_root_agent.rb +139 -0
  118. data/test/test_test_drivers.rb +135 -0
  119. metadata +28 -8
  120. data/test/plugin/test_parser_base.rb +0 -32
@@ -48,10 +48,19 @@ module Fluent
48
48
  }
49
49
 
50
50
  PARSER_PARAMS = {
51
- "format" => "@type",
51
+ "format" => nil,
52
+ "types" => nil,
53
+ "types_delimiter" => nil,
54
+ "types_label_delimiter" => nil,
55
+ "keys" => "keys", # CSVParser, TSVParser (old ValuesParser)
52
56
  "time_key" => "time_key",
53
57
  "time_format" => "time_format",
58
+ "localtim" => nil,
59
+ "utc" => nil,
54
60
  "delimiter" => "delimiter",
61
+ "keep_time_key" => "keep_time_key",
62
+ "null_empty_string" => "null_empty_string",
63
+ "null_value_pattern" => "null_value_pattern",
55
64
  "json_parser" => "json_parser", # JSONParser
56
65
  "label_delimiter" => "label_delimiter", # LabeledTSVParser
57
66
  "format_firstline" => "format_firstline", # MultilineParser
@@ -62,19 +71,29 @@ module Fluent
62
71
 
63
72
  INJECT_PARAMS = {
64
73
  "include_time_key" => nil,
65
- "time_key" => "time_key",
66
- "time_format" => "time_format",
67
- "timezone" => "timezone",
74
+ "time_key" => "time_key",
75
+ "time_format" => "time_format",
76
+ "timezone" => "timezone",
68
77
  "include_tag_key" => nil,
69
78
  "tag_key" => "tag_key",
70
79
  "localtime" => nil,
71
80
  "utc" => nil,
72
81
  }
73
82
 
83
+ EXTRACT_PARAMS = {
84
+ "time_key" => "time_key",
85
+ "time_format" => "time_format",
86
+ "timezone" => "timezone",
87
+ "tag_key" => "tag_key",
88
+ "localtime" => nil,
89
+ "utc" => nil,
90
+ }
91
+
74
92
  FORMATTER_PARAMS = {
75
93
  "format" => "@type",
76
94
  "delimiter" => "delimiter",
77
95
  "force_quotes" => "force_quotes", # CsvFormatter
96
+ "keys" => "keys", # TSVFormatter
78
97
  "fields" => "fields", # CsvFormatter
79
98
  "json_parser" => "json_parser", # JSONFormatter
80
99
  "label_delimiter" => "label_delimiter", # LabeledTSVFormatter
@@ -95,6 +114,8 @@ module Fluent
95
114
  compat_parameters_buffer(conf, **kwargs)
96
115
  when :inject
97
116
  compat_parameters_inject(conf)
117
+ when :extract
118
+ compat_parameters_extract(conf)
98
119
  when :parser
99
120
  compat_parameters_parser(conf)
100
121
  when :formatter
@@ -167,6 +188,7 @@ module Fluent
167
188
  hash['time_type'] ||= 'string'
168
189
  end
169
190
  if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
191
+ hash['time_key'] ||= 'time'
170
192
  hash['time_type'] = 'unixtime'
171
193
  end
172
194
  if conf.has_key?('localtime') || conf.has_key?('utc')
@@ -193,6 +215,40 @@ module Fluent
193
215
  conf
194
216
  end
195
217
 
218
+ def compat_parameters_extract(conf)
219
+ return unless conf.elements('extract').empty?
220
+ return if EXTRACT_PARAMS.keys.all?{|k| !conf.has_key?(k) } && !conf.has_key?('format')
221
+
222
+ # TODO: warn obsolete parameters if these are deprecated
223
+ hash = compat_parameters_copy_to_subsection_attributes(conf, EXTRACT_PARAMS)
224
+
225
+ if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
226
+ hash['time_key'] ||= 'time'
227
+ hash['time_type'] = 'unixtime'
228
+ elsif conf.has_key?('format') && conf["format"].start_with?("/") && conf["format"].end_with?("/") # old-style regexp parser
229
+ hash['time_key'] ||= 'time'
230
+ hash['time_type'] ||= 'string'
231
+ end
232
+ if conf.has_key?('localtime') || conf.has_key?('utc')
233
+ if conf.has_key?('localtime') && conf.has_key?('utc')
234
+ raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
235
+ elsif conf.has_key?('localtime')
236
+ hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
237
+ elsif conf.has_key?('utc')
238
+ hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
239
+ # Specifying "localtime false" means using UTC in TimeFormatter
240
+ # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
241
+ # There are difference between "Z" and "+0000" in timezone formatting.
242
+ # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
243
+ end
244
+ end
245
+
246
+ e = Fluent::Config::Element.new('extract', '', hash, [])
247
+ conf.elements << e
248
+
249
+ conf
250
+ end
251
+
196
252
  def compat_parameters_parser(conf)
197
253
  return unless conf.elements('parse').empty?
198
254
  return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }
@@ -200,6 +256,39 @@ module Fluent
200
256
  # TODO: warn obsolete parameters if these are deprecated
201
257
  hash = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)
202
258
 
259
+ if conf["format"]
260
+ if conf["format"].start_with?("/") && conf["format"].end_with?("/")
261
+ hash["@type"] = "regexp"
262
+ hash["expression"] = conf["format"][1..-2]
263
+ else
264
+ hash["@type"] = conf["format"]
265
+ end
266
+ end
267
+
268
+ if conf["types"]
269
+ delimiter = conf["types_delimiter"] || ','
270
+ label_delimiter = conf["types_label_delimiter"] || ':'
271
+ types = {}
272
+ conf['types'].split(delimiter).each do |pair|
273
+ key, value = pair.split(label_delimiter, 2)
274
+ types[key] = value
275
+ end
276
+ hash["types"] = JSON.dump(types)
277
+ end
278
+ if conf.has_key?('localtime') || conf.has_key?('utc')
279
+ if conf.has_key?('localtime') && conf.has_key?('utc')
280
+ raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
281
+ elsif conf.has_key?('localtime')
282
+ hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
283
+ elsif conf.has_key?('utc')
284
+ hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
285
+ # Specifying "localtime false" means using UTC in TimeFormatter
286
+ # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
287
+ # There are difference between "Z" and "+0000" in timezone formatting.
288
+ # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
289
+ end
290
+ end
291
+
203
292
  e = Fluent::Config::Element.new('parse', '', hash, [])
204
293
  conf.elements << e
205
294
 
@@ -26,10 +26,14 @@ module Fluent
26
26
 
27
27
  def router
28
28
  @_event_emitter_used_actually = true
29
+ if @_event_emitter_lazy_init
30
+ @router = @primary_instance.router
31
+ end
29
32
  @router
30
33
  end
31
34
 
32
35
  def router=(r)
36
+ # not recommended now...
33
37
  @router = r
34
38
  end
35
39
 
@@ -44,14 +48,23 @@ module Fluent
44
48
  def event_emitter_router(label_name)
45
49
  if label_name
46
50
  Engine.root_agent.find_label(label_name).event_router
51
+ elsif self.respond_to?(:as_secondary) && self.as_secondary
52
+ if @primary_instance.has_router?
53
+ @_event_emitter_lazy_init = true
54
+ nil # primary plugin's event router is not initialized yet, here.
55
+ else
56
+ @primary_instance.context_router
57
+ end
47
58
  else
48
- Engine.root_agent.event_router
59
+ # `Engine.root_agent.event_router` is for testing
60
+ self.context_router || Engine.root_agent.event_router
49
61
  end
50
62
  end
51
63
 
52
64
  def initialize
53
65
  super
54
66
  @_event_emitter_used_actually = false
67
+ @_event_emitter_lazy_init = false
55
68
  @router = nil
56
69
  end
57
70
 
@@ -30,12 +30,16 @@ module Fluent
30
30
  # terminate: initialize internal state
31
31
 
32
32
  EVENT_LOOP_RUN_DEFAULT_TIMEOUT = 0.5
33
+ EVENT_LOOP_SHUTDOWN_TIMEOUT = 5
34
+ EVENT_LOOP_CLOCK_ID = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
33
35
 
34
36
  attr_reader :_event_loop # for tests
35
37
 
36
38
  def event_loop_attach(watcher)
37
39
  @_event_loop_mutex.synchronize do
38
40
  @_event_loop.attach(watcher)
41
+ @_event_loop_attached_watchers << watcher
42
+ watcher
39
43
  end
40
44
  end
41
45
 
@@ -58,6 +62,7 @@ module Fluent
58
62
  @_event_loop_mutex = Mutex.new
59
63
  # plugin MAY configure loop run timeout in #configure
60
64
  @_event_loop_run_timeout = EVENT_LOOP_RUN_DEFAULT_TIMEOUT
65
+ @_event_loop_attached_watchers = []
61
66
  end
62
67
 
63
68
  def start
@@ -65,19 +70,32 @@ module Fluent
65
70
 
66
71
  # event loop does not run here, so mutex lock is not required
67
72
  thread_create :event_loop do
68
- default_watcher = DefaultWatcher.new
69
- @_event_loop.attach(default_watcher)
70
- @_event_loop_running = true
71
- @_event_loop.run(@_event_loop_run_timeout) # this method blocks
72
- @_event_loop_running = false
73
+ begin
74
+ default_watcher = DefaultWatcher.new
75
+ event_loop_attach(default_watcher)
76
+ @_event_loop_running = true
77
+ @_event_loop.run(@_event_loop_run_timeout) # this method blocks
78
+ ensure
79
+ @_event_loop_running = false
80
+ end
73
81
  end
74
82
  end
75
83
 
76
84
  def shutdown
77
85
  @_event_loop_mutex.synchronize do
78
- @_event_loop.watchers.each {|w| w.detach if w.attached? }
86
+ @_event_loop_attached_watchers.reverse.each do |w|
87
+ if w.attached?
88
+ w.detach
89
+ end
90
+ end
79
91
  end
92
+ timeout_at = Process.clock_gettime(EVENT_LOOP_CLOCK_ID) + EVENT_LOOP_SHUTDOWN_TIMEOUT
80
93
  while @_event_loop_running
94
+ if Process.clock_gettime(EVENT_LOOP_CLOCK_ID) >= timeout_at
95
+ log.warn "event loop does NOT exit until hard timeout."
96
+ raise "event loop does NOT exit until hard timeout." if @under_plugin_development
97
+ break
98
+ end
81
99
  sleep 0.1
82
100
  end
83
101
 
@@ -25,7 +25,8 @@ module Fluent
25
25
  return nil unless @_extract_enabled
26
26
 
27
27
  if @_extract_tag_key && record.has_key?(@_extract_tag_key)
28
- return record[@_extract_tag_key].to_s
28
+ v = @_extract_keep_tag_key ? record[@_extract_tag_key] : record.delete(@_extract_tag_key)
29
+ return v.to_s
29
30
  end
30
31
 
31
32
  nil
@@ -35,7 +36,8 @@ module Fluent
35
36
  return nil unless @_extract_enabled
36
37
 
37
38
  if @_extract_time_key && record.has_key?(@_extract_time_key)
38
- return @_extract_time_parser.call(record[@_extract_time_key])
39
+ v = @_extract_keep_time_key ? record[@_extract_time_key] : record.delete(@_extract_time_key)
40
+ return @_extract_time_parser.call(v)
39
41
  end
40
42
 
41
43
  nil
@@ -45,7 +47,9 @@ module Fluent
45
47
  include Fluent::Configurable
46
48
  config_section :extract, required: false, multi: false, param_name: :extract_config do
47
49
  config_param :tag_key, :string, default: nil
50
+ config_param :keep_tag_key, :bool, default: false
48
51
  config_param :time_key, :string, default: nil
52
+ config_param :keep_time_key, :bool, default: false
49
53
 
50
54
  # To avoid defining :time_type twice
51
55
  config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
@@ -64,7 +68,9 @@ module Fluent
64
68
  super
65
69
  @_extract_enabled = false
66
70
  @_extract_tag_key = nil
71
+ @_extract_keep_tag_key = nil
67
72
  @_extract_time_key = nil
73
+ @_extract_keep_time_key = nil
68
74
  @_extract_time_parser = nil
69
75
  end
70
76
 
@@ -73,15 +79,21 @@ module Fluent
73
79
 
74
80
  if @extract_config
75
81
  @_extract_tag_key = @extract_config.tag_key
82
+ @_extract_keep_tag_key = @extract_config.keep_tag_key
76
83
  @_extract_time_key = @extract_config.time_key
77
84
  if @_extract_time_key
85
+ @_extract_keep_time_key = @extract_config.keep_time_key
78
86
  @_extract_time_parser = case @extract_config.time_type
79
- when :float then ->(v){ Fluent::EventTime.new(v.to_i, ((v.to_f - v.to_i) * 1_000_000_000).to_i) }
80
- when :unixtime then ->(v){ Fluent::EventTime.new(v.to_i, 0) }
87
+ when :float then Fluent::NumericTimeParser.new(:float)
88
+ when :unixtime then Fluent::NumericTimeParser.new(:unixtime)
81
89
  else
82
90
  localtime = @extract_config.localtime && !@extract_config.utc
83
91
  Fluent::TimeParser.new(@extract_config.time_format, localtime, @extract_config.timezone)
84
92
  end
93
+ else
94
+ if @extract_config.time_format
95
+ log.warn "'time_format' specified without 'time_key', will be ignored"
96
+ end
85
97
  end
86
98
 
87
99
  @_extract_enabled = @_extract_tag_key || @_extract_time_key
@@ -24,7 +24,7 @@ module Fluent
24
24
  module Formatter
25
25
  def formatter_create(usage: '', type: nil, conf: nil, default_type: nil)
26
26
  formatter = @_formatters[usage]
27
- return formatter if formatter
27
+ return formatter if formatter && !type && !conf
28
28
 
29
29
  type = if type
30
30
  type
@@ -61,9 +61,9 @@ module Fluent
61
61
  module FormatterParams
62
62
  include Fluent::Configurable
63
63
  # minimum section definition to instantiate formatter plugin instances
64
- config_section :format, required: false, multi: true, param_name: :formatter_configs do
64
+ config_section :format, required: false, multi: true, init: true, param_name: :formatter_configs do
65
65
  config_argument :usage, :string, default: ''
66
- config_param :@type, :string
66
+ config_param :@type, :string # config_set_default required for :@type
67
67
  end
68
68
  end
69
69
 
@@ -82,15 +82,13 @@ module Fluent
82
82
  def configure(conf)
83
83
  super
84
84
 
85
- if @formatter_configs
86
- @formatter_configs.each do |section|
87
- if @_formatters[section.usage]
88
- raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
89
- end
90
- formatter = Plugin.new_formatter(section[:@type], parent: self)
91
- formatter.configure(section.corresponding_config_element)
92
- @_formatters[section.usage] = formatter
85
+ @formatter_configs.each do |section|
86
+ if @_formatters[section.usage]
87
+ raise Fluent::ConfigError, "duplicated formatter configured: #{section.usage}"
93
88
  end
89
+ formatter = Plugin.new_formatter(section[:@type], parent: self)
90
+ formatter.configure(section.corresponding_config_element)
91
+ @_formatters[section.usage] = formatter
94
92
  end
95
93
  end
96
94
 
@@ -97,9 +97,20 @@ module Fluent
97
97
  if @inject_config
98
98
  @_inject_hostname_key = @inject_config.hostname_key
99
99
  if @_inject_hostname_key
100
+ if self.respond_to?(:buffer_config)
101
+ # Output plugin cannot use "hostname"(specified by @hostname_key),
102
+ # injected by this plugin helper, in chunk keys.
103
+ # This plugin helper works in `#format` (in many cases), but modified record
104
+ # don't have any side effect in chunking of output plugin.
105
+ if self.buffer_config.chunk_keys.include?(@_inject_hostname_key)
106
+ log.error "Use filters to inject hostname to use it in buffer chunking."
107
+ raise Fluent::ConfigError, "the key specified by 'hostname_key' in <inject> cannot be used in buffering chunk key."
108
+ end
109
+ end
110
+
100
111
  @_inject_hostname = @inject_config.hostname
101
112
  unless @_inject_hostname
102
- @_inject_hostname = Socket.gethostname
113
+ @_inject_hostname = ::Socket.gethostname
103
114
  log.info "using hostname for specified field", host_key: @_inject_hostname_key, host_name: @_inject_hostname
104
115
  end
105
116
  end
@@ -113,6 +124,10 @@ module Fluent
113
124
  localtime = @inject_config.localtime && !@inject_config.utc
114
125
  Fluent::TimeFormatter.new(@inject_config.time_format, localtime, @inject_config.timezone)
115
126
  end
127
+ else
128
+ if @inject_config.time_format
129
+ log.warn "'time_format' specified without 'time_key', will be ignored"
130
+ end
116
131
  end
117
132
 
118
133
  @_inject_enabled = @_inject_hostname_key || @_inject_tag_key || @_inject_time_key
@@ -24,7 +24,7 @@ module Fluent
24
24
  module Parser
25
25
  def parser_create(usage: '', type: nil, conf: nil, default_type: nil)
26
26
  parser = @_parsers[usage]
27
- return parser if parser
27
+ return parser if parser && !type && !conf
28
28
 
29
29
  type = if type
30
30
  type
@@ -61,9 +61,9 @@ module Fluent
61
61
  module ParserParams
62
62
  include Fluent::Configurable
63
63
  # minimum section definition to instantiate parser plugin instances
64
- config_section :parse, required: false, multi: true, param_name: :parser_configs do
64
+ config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
65
65
  config_argument :usage, :string, default: ''
66
- config_param :@type, :string
66
+ config_param :@type, :string # config_set_default required for :@type
67
67
  end
68
68
  end
69
69
 
@@ -0,0 +1,494 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin_helper/event_loop'
18
+
19
+ require 'serverengine'
20
+ require 'cool.io'
21
+ require 'socket'
22
+ require 'ipaddr'
23
+ require 'fcntl'
24
+
25
+ require_relative 'socket_option'
26
+
27
+ module Fluent
28
+ module PluginHelper
29
+ module Server
30
+ include Fluent::PluginHelper::EventLoop
31
+ include Fluent::PluginHelper::SocketOption
32
+
33
+ # This plugin helper doesn't support these things for now:
34
+ # * SSL/TLS (TBD)
35
+ # * TCP/TLS keepalive
36
+
37
+ # stop : [-]
38
+ # shutdown : detach server event handler from event loop (event_loop)
39
+ # close : close listening sockets
40
+ # terminate: remote all server instances
41
+
42
+ attr_reader :_servers # for tests
43
+
44
+ def server_wait_until_start
45
+ # event_loop_wait_until_start works well for this
46
+ end
47
+
48
+ def server_wait_until_stop
49
+ sleep 0.1 while @_servers.any?{|si| si.server.attached? }
50
+ @_servers.each{|si| si.server.close rescue nil }
51
+ end
52
+
53
+ PROTOCOLS = [:tcp, :udp, :tls, :unix]
54
+ CONNECTION_PROTOCOLS = [:tcp, :tls, :unix]
55
+
56
+ # server_create_connection(:title, @port) do |conn|
57
+ # # on connection
58
+ # source_addr = conn.remote_host
59
+ # source_port = conn.remote_port
60
+ # conn.data do |data|
61
+ # # on data
62
+ # conn.write resp # ...
63
+ # conn.close
64
+ # end
65
+ # end
66
+ def server_create_connection(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, backlog: nil, **socket_options, &block)
67
+ raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
68
+ raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
69
+ raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
70
+ raise ArgumentError, "BUG: cannot create connection for UDP" unless CONNECTION_PROTOCOLS.include?(proto)
71
+
72
+ raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
73
+ raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
74
+
75
+ if proto == :tcp || proto == :tls # default linger_timeout only for server
76
+ socket_options[:linger_timeout] ||= 0
77
+ end
78
+
79
+ socket_option_validate!(proto, **socket_options)
80
+ socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
81
+
82
+ case proto
83
+ when :tcp
84
+ server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
85
+ when :tls
86
+ raise ArgumentError, "BUG: certopts (certificate options) not specified for TLS" unless certopts
87
+ # server_certopts_validate!(certopts)
88
+ # sock = server_create_tls_socket(shared, bind, port)
89
+ # server = nil # ...
90
+ raise "not implemented yet"
91
+ when :unix
92
+ raise "not implemented yet"
93
+ else
94
+ raise "unknown protocol #{proto}"
95
+ end
96
+
97
+ server_attach(title, proto, port, bind, shared, server)
98
+ end
99
+
100
+ # server_create(:title, @port) do |data|
101
+ # # ...
102
+ # end
103
+ # server_create(:title, @port) do |data, conn|
104
+ # # ...
105
+ # end
106
+ # server_create(:title, @port, proto: :udp, max_bytes: 2048) do |data, sock|
107
+ # sock.remote_host
108
+ # sock.remote_port
109
+ # # ...
110
+ # end
111
+ def server_create(title, port, proto: :tcp, bind: '0.0.0.0', shared: true, socket: nil, backlog: nil, max_bytes: nil, flags: 0, **socket_options, &callback)
112
+ raise ArgumentError, "BUG: title must be a symbol" unless title && title.is_a?(Symbol)
113
+ raise ArgumentError, "BUG: port must be an integer" unless port && port.is_a?(Integer)
114
+ raise ArgumentError, "BUG: invalid protocol name" unless PROTOCOLS.include?(proto)
115
+
116
+ raise ArgumentError, "BUG: socket option is available only for udp" if socket && proto != :udp
117
+
118
+ raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
119
+ raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
120
+
121
+ if proto == :tcp || proto == :tls # default linger_timeout only for server
122
+ socket_options[:linger_timeout] ||= 0
123
+ end
124
+
125
+ unless socket
126
+ socket_option_validate!(proto, **socket_options)
127
+ socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
128
+ end
129
+
130
+ if proto != :tcp && proto != :tls && proto != :unix # options to listen/accept connections
131
+ raise ArgumentError, "BUG: backlog is available for tcp/tls" if backlog
132
+ end
133
+ if proto != :udp # UDP options
134
+ raise ArgumentError, "BUG: max_bytes is available only for udp" if max_bytes
135
+ raise ArgumentError, "BUG: flags is available only for udp" if flags != 0
136
+ end
137
+
138
+ case proto
139
+ when :tcp
140
+ server = server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter) do |conn|
141
+ conn.data(&callback)
142
+ end
143
+ when :tls
144
+ raise "not implemented yet"
145
+ when :udp
146
+ raise ArgumentError, "BUG: max_bytes must be specified for UDP" unless max_bytes
147
+ if socket
148
+ sock = socket
149
+ close_socket = false
150
+ else
151
+ sock = server_create_udp_socket(shared, bind, port)
152
+ socket_option_setter.call(sock)
153
+ close_socket = true
154
+ end
155
+ server = EventHandler::UDPServer.new(sock, max_bytes, flags, close_socket, @log, @under_plugin_development, &callback)
156
+ when :unix
157
+ raise "not implemented yet"
158
+ else
159
+ raise "BUG: unknown protocol #{proto}"
160
+ end
161
+
162
+ server_attach(title, proto, port, bind, shared, server)
163
+ end
164
+
165
+ def server_create_tcp(title, port, **kwargs, &callback)
166
+ server_create(title, port, proto: :tcp, **kwargs, &callback)
167
+ end
168
+
169
+ def server_create_udp(title, port, **kwargs, &callback)
170
+ server_create(title, port, proto: :udp, **kwargs, &callback)
171
+ end
172
+
173
+ def server_create_tls(title, port, **kwargs, &callback)
174
+ server_create(title, port, proto: :tls, **kwargs, &callback)
175
+ end
176
+
177
+ def server_create_unix(title, port, **kwargs, &callback)
178
+ server_create(title, port, proto: :unix, **kwargs, &callback)
179
+ end
180
+
181
+ ServerInfo = Struct.new(:title, :proto, :port, :bind, :shared, :server)
182
+
183
+ def server_attach(title, proto, port, bind, shared, server)
184
+ @_servers << ServerInfo.new(title, proto, port, bind, shared, server)
185
+ event_loop_attach(server)
186
+ end
187
+
188
+ def server_create_for_tcp_connection(shared, bind, port, backlog, socket_option_setter, &block)
189
+ sock = server_create_tcp_socket(shared, bind, port)
190
+ socket_option_setter.call(sock)
191
+ close_callback = ->(conn){ @_server_mutex.synchronize{ @_server_connections.delete(conn) } }
192
+ server = Coolio::TCPServer.new(sock, nil, EventHandler::TCPServer, socket_option_setter, close_callback, @log, @under_plugin_development, block) do |conn|
193
+ @_server_mutex.synchronize do
194
+ @_server_connections << conn
195
+ end
196
+ end
197
+ server.listen(backlog) if backlog
198
+ server
199
+ end
200
+
201
+ def initialize
202
+ super
203
+ @_servers = []
204
+ @_server_connections = []
205
+ @_server_mutex = Mutex.new
206
+ end
207
+
208
+ def shutdown
209
+ @_server_connections.each do |conn|
210
+ conn.close rescue nil
211
+ end
212
+ @_server_mutex.synchronize do
213
+ @_servers.each do |si|
214
+ si.server.detach if si.server.attached?
215
+ end
216
+ end
217
+
218
+ super
219
+ end
220
+
221
+ def close
222
+ @_server_connections.each do |conn|
223
+ conn.close rescue nil
224
+ end
225
+ @_server_mutex.synchronize do
226
+ @_servers.each do |si|
227
+ si.server.close rescue nil
228
+ end
229
+ end
230
+ super
231
+ end
232
+
233
+ def terminate
234
+ @_servers = []
235
+ super
236
+ end
237
+
238
+ def server_certopts_validate!(certopts)
239
+ raise "not implemented yet"
240
+ end
241
+
242
+ def server_socket_manager_client
243
+ socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
244
+ if Fluent.windows?
245
+ socket_manager_path = socket_manager_path.to_i
246
+ end
247
+ ServerEngine::SocketManager::Client.new(socket_manager_path)
248
+ end
249
+
250
+ def server_create_tcp_socket(shared, bind, port)
251
+ sock = if shared
252
+ server_socket_manager_client.listen_tcp(bind, port)
253
+ else
254
+ TCPServer.new(bind, port) # this method call can create sockets for AF_INET6
255
+ end
256
+ # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
257
+ sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
258
+ sock
259
+ end
260
+
261
+ def server_create_udp_socket(shared, bind, port)
262
+ sock = if shared
263
+ server_socket_manager_client.listen_udp(bind, port)
264
+ else
265
+ family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
266
+ usock = UDPSocket.new(family)
267
+ usock.bind(bind, port)
268
+ usock
269
+ end
270
+ # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
271
+ sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
272
+ sock
273
+ end
274
+
275
+ def server_create_tls_socket(shared, bind, port)
276
+ raise "not implemented yet"
277
+ end
278
+
279
+ class CallbackSocket
280
+ def initialize(server_type, sock, enabled_events = [], close_socket: true)
281
+ @server_type = server_type
282
+ @sock = sock
283
+ @enabled_events = enabled_events
284
+ @close_socket = close_socket
285
+ end
286
+
287
+ def remote_addr
288
+ @sock.peeraddr[3]
289
+ end
290
+
291
+ def remote_host
292
+ @sock.peeraddr[2]
293
+ end
294
+
295
+ def remote_port
296
+ @sock.peeraddr[1]
297
+ end
298
+
299
+ def send(data, flags = 0)
300
+ @sock.send(data, flags)
301
+ end
302
+
303
+ def write(data)
304
+ raise "not implemented here"
305
+ end
306
+
307
+ def close
308
+ @sock.close if @close_socket
309
+ end
310
+
311
+ def data(&callback)
312
+ on(:data, &callback)
313
+ end
314
+
315
+ def on(event, &callback)
316
+ raise "BUG: this event is disabled for #{@server_type}: #{event}" unless @enabled_events.include?(event)
317
+ case event
318
+ when :data
319
+ @sock.data(&callback)
320
+ when :write_complete
321
+ cb = ->(){ callback.call(self) }
322
+ @sock.on_write_complete(&cb)
323
+ when :close
324
+ cb = ->(){ callback.call(self) }
325
+ @sock.on_close(&cb)
326
+ else
327
+ raise "BUG: unknown event: #{event}"
328
+ end
329
+ end
330
+ end
331
+
332
+ class TCPCallbackSocket < CallbackSocket
333
+ def initialize(sock)
334
+ super("tcp", sock, [:data, :write_complete, :close])
335
+ end
336
+
337
+ def write(data)
338
+ @sock.write(data)
339
+ end
340
+ end
341
+
342
+ class UDPCallbackSocket < CallbackSocket
343
+ def initialize(sock, peeraddr, **kwargs)
344
+ super("udp", sock, [], **kwargs)
345
+ @peeraddr = peeraddr
346
+ end
347
+
348
+ def remote_addr
349
+ @peeraddr[3]
350
+ end
351
+
352
+ def remote_host
353
+ @peeraddr[2]
354
+ end
355
+
356
+ def remote_port
357
+ @peeraddr[1]
358
+ end
359
+
360
+ def write(data)
361
+ @sock.send(data, 0, @peeraddr[3], @peeraddr[1])
362
+ end
363
+ end
364
+
365
+ module EventHandler
366
+ class UDPServer < Coolio::IO
367
+ def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
368
+ raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
369
+
370
+ super(sock)
371
+
372
+ @sock = sock
373
+ @max_bytes = max_bytes
374
+ @flags = flags
375
+ @close_socket = close_socket
376
+ @log = log
377
+ @under_plugin_development = under_plugin_development
378
+ @callback = callback
379
+
380
+ on_readable_impl = case @callback.arity
381
+ when 1 then :on_readable_without_sock
382
+ when 2 then :on_readable_with_sock
383
+ else
384
+ raise "BUG: callback block must have 1 or 2 arguments"
385
+ end
386
+ self.define_singleton_method(:on_readable, method(on_readable_impl))
387
+ end
388
+
389
+ def on_readable_without_sock
390
+ begin
391
+ data = @sock.recv(@max_bytes, @flags)
392
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET
393
+ return
394
+ end
395
+ @callback.call(data)
396
+ rescue => e
397
+ @log.error "unexpected error in processing UDP data", error: e
398
+ @log.error_backtrace
399
+ raise if @under_plugin_development
400
+ end
401
+
402
+ def on_readable_with_sock
403
+ begin
404
+ data, addr = @sock.recvfrom(@max_bytes)
405
+ rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET
406
+ return
407
+ end
408
+ @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
409
+ rescue => e
410
+ @log.error "unexpected error in processing UDP data", error: e
411
+ @log.error_backtrace
412
+ raise if @under_plugin_development
413
+ end
414
+ end
415
+
416
+ class TCPServer < Coolio::TCPSocket
417
+ def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
418
+ raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
419
+
420
+ socket_option_setter.call(sock)
421
+
422
+ @_handler_socket = sock
423
+ super(sock)
424
+
425
+ @log = log
426
+ @under_plugin_development = under_plugin_development
427
+
428
+ @connect_callback = connect_callback
429
+ @data_callback = nil
430
+ @close_callback = close_callback
431
+
432
+ @callback_connection = nil
433
+ @closing = false
434
+
435
+ @mutex = Mutex.new # to serialize #write and #close
436
+ end
437
+
438
+ def data(&callback)
439
+ raise "data callback can be registered just once, but registered twice" if self.singleton_methods.include?(:on_read)
440
+ @data_callback = callback
441
+ on_read_impl = case callback.arity
442
+ when 1 then :on_read_without_connection
443
+ when 2 then :on_read_with_connection
444
+ else
445
+ raise "BUG: callback block must have 1 or 2 arguments"
446
+ end
447
+ self.define_singleton_method(:on_read, method(on_read_impl))
448
+ end
449
+
450
+ def write(data)
451
+ @mutex.synchronize do
452
+ super
453
+ end
454
+ end
455
+
456
+ def on_connect
457
+ @callback_connection = TCPCallbackSocket.new(self)
458
+ @connect_callback.call(@callback_connection)
459
+ unless @data_callback
460
+ raise "connection callback must call #data to set data callback"
461
+ end
462
+ end
463
+
464
+ def on_read_without_connection(data)
465
+ @data_callback.call(data)
466
+ rescue => e
467
+ @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
468
+ @log.error_backtrace
469
+ close(true) rescue nil
470
+ raise if @under_plugin_development
471
+ end
472
+
473
+ def on_read_with_connection(data)
474
+ @data_callback.call(data, @callback_connection)
475
+ rescue => e
476
+ @log.error "unexpected error on reading data", host: remote_host, port: remote_port, error: e
477
+ @log.error_backtrace
478
+ close(true) rescue nil
479
+ raise if @under_plugin_development
480
+ end
481
+
482
+ def close
483
+ @mutex.synchronize do
484
+ return if @closing
485
+ @closing = true
486
+ @close_callback.call(self)
487
+ super
488
+ end
489
+ end
490
+ end
491
+ end
492
+ end
493
+ end
494
+ end