fluentd 0.14.11 → 0.14.12

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (119) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -5
  3. data/ChangeLog +54 -2
  4. data/example/in_dummy_blocks.conf +17 -0
  5. data/example/in_forward_tls.conf +14 -0
  6. data/example/in_forward_workers.conf +21 -0
  7. data/example/logevents.conf +25 -0
  8. data/example/out_forward_heartbeat_none.conf +16 -0
  9. data/example/out_forward_tls.conf +18 -0
  10. data/example/suppress_config_dump.conf +7 -0
  11. data/lib/fluent/agent.rb +3 -32
  12. data/lib/fluent/clock.rb +62 -0
  13. data/lib/fluent/command/fluentd.rb +12 -0
  14. data/lib/fluent/compat/input.rb +10 -1
  15. data/lib/fluent/compat/output.rb +40 -1
  16. data/lib/fluent/config/configure_proxy.rb +30 -7
  17. data/lib/fluent/config/section.rb +4 -0
  18. data/lib/fluent/config/types.rb +2 -2
  19. data/lib/fluent/configurable.rb +31 -5
  20. data/lib/fluent/engine.rb +61 -12
  21. data/lib/fluent/event_router.rb +6 -0
  22. data/lib/fluent/load.rb +0 -1
  23. data/lib/fluent/log.rb +118 -42
  24. data/lib/fluent/match.rb +37 -0
  25. data/lib/fluent/plugin.rb +25 -3
  26. data/lib/fluent/plugin/base.rb +4 -0
  27. data/lib/fluent/plugin/buf_file.rb +38 -14
  28. data/lib/fluent/plugin/buffer.rb +20 -20
  29. data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
  30. data/lib/fluent/plugin/compressable.rb +1 -0
  31. data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
  32. data/lib/fluent/plugin/formatter_csv.rb +4 -1
  33. data/lib/fluent/plugin/formatter_hash.rb +5 -1
  34. data/lib/fluent/plugin/formatter_json.rb +10 -0
  35. data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
  36. data/lib/fluent/plugin/in_dummy.rb +4 -0
  37. data/lib/fluent/plugin/in_exec.rb +4 -0
  38. data/lib/fluent/plugin/in_forward.rb +11 -3
  39. data/lib/fluent/plugin/in_gc_stat.rb +4 -0
  40. data/lib/fluent/plugin/in_http.rb +4 -0
  41. data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
  42. data/lib/fluent/plugin/in_object_space.rb +4 -1
  43. data/lib/fluent/plugin/in_syslog.rb +4 -0
  44. data/lib/fluent/plugin/in_tail.rb +193 -116
  45. data/lib/fluent/plugin/in_tcp.rb +5 -1
  46. data/lib/fluent/plugin/in_udp.rb +4 -0
  47. data/lib/fluent/plugin/input.rb +4 -0
  48. data/lib/fluent/plugin/out_copy.rb +4 -0
  49. data/lib/fluent/plugin/out_exec.rb +4 -0
  50. data/lib/fluent/plugin/out_exec_filter.rb +4 -0
  51. data/lib/fluent/plugin/out_file.rb +70 -30
  52. data/lib/fluent/plugin/out_forward.rb +132 -28
  53. data/lib/fluent/plugin/out_null.rb +10 -0
  54. data/lib/fluent/plugin/out_relabel.rb +4 -0
  55. data/lib/fluent/plugin/out_roundrobin.rb +4 -0
  56. data/lib/fluent/plugin/out_secondary_file.rb +5 -0
  57. data/lib/fluent/plugin/out_stdout.rb +5 -0
  58. data/lib/fluent/plugin/output.rb +18 -9
  59. data/lib/fluent/plugin/storage_local.rb +25 -2
  60. data/lib/fluent/plugin_helper/cert_option.rb +159 -0
  61. data/lib/fluent/plugin_helper/child_process.rb +6 -6
  62. data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
  63. data/lib/fluent/plugin_helper/event_loop.rb +29 -4
  64. data/lib/fluent/plugin_helper/inject.rb +14 -1
  65. data/lib/fluent/plugin_helper/server.rb +275 -31
  66. data/lib/fluent/plugin_helper/socket.rb +144 -4
  67. data/lib/fluent/plugin_helper/socket_option.rb +2 -17
  68. data/lib/fluent/plugin_helper/storage.rb +7 -1
  69. data/lib/fluent/plugin_helper/thread.rb +16 -4
  70. data/lib/fluent/registry.rb +26 -9
  71. data/lib/fluent/root_agent.rb +7 -3
  72. data/lib/fluent/supervisor.rb +37 -15
  73. data/lib/fluent/system_config.rb +37 -10
  74. data/lib/fluent/test.rb +2 -0
  75. data/lib/fluent/test/driver/base.rb +24 -26
  76. data/lib/fluent/test/helpers.rb +21 -0
  77. data/lib/fluent/version.rb +1 -1
  78. data/test/command/test_fluentd.rb +274 -4
  79. data/test/config/test_configurable.rb +154 -0
  80. data/test/config/test_configure_proxy.rb +180 -1
  81. data/test/config/test_system_config.rb +10 -0
  82. data/test/config/test_types.rb +1 -0
  83. data/test/plugin/test_base.rb +4 -0
  84. data/test/plugin/test_buf_file.rb +241 -9
  85. data/test/plugin/test_buffer.rb +11 -11
  86. data/test/plugin/test_buffer_file_chunk.rb +6 -6
  87. data/test/plugin/test_compressable.rb +3 -0
  88. data/test/plugin/test_filter.rb +4 -0
  89. data/test/plugin/test_filter_record_transformer.rb +20 -0
  90. data/test/plugin/test_formatter_csv.rb +9 -0
  91. data/test/plugin/test_formatter_hash.rb +35 -0
  92. data/test/plugin/test_formatter_json.rb +8 -0
  93. data/test/plugin/test_formatter_ltsv.rb +7 -0
  94. data/test/plugin/test_in_dummy.rb +7 -3
  95. data/test/plugin/test_in_monitor_agent.rb +43 -5
  96. data/test/plugin/test_in_tail.rb +97 -4
  97. data/test/plugin/test_input.rb +4 -0
  98. data/test/plugin/test_out_file.rb +46 -7
  99. data/test/plugin/test_out_forward.rb +59 -7
  100. data/test/plugin/test_output.rb +10 -4
  101. data/test/plugin/test_output_as_buffered.rb +37 -25
  102. data/test/plugin/test_output_as_buffered_compress.rb +1 -1
  103. data/test/plugin/test_output_as_buffered_retries.rb +6 -6
  104. data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
  105. data/test/plugin/test_storage_local.rb +40 -1
  106. data/test/plugin_helper/test_child_process.rb +29 -28
  107. data/test/plugin_helper/test_compat_parameters.rb +1 -1
  108. data/test/plugin_helper/test_inject.rb +27 -9
  109. data/test/plugin_helper/test_server.rb +822 -50
  110. data/test/plugin_helper/test_storage.rb +11 -0
  111. data/test/plugin_helper/test_timer.rb +1 -0
  112. data/test/test_clock.rb +164 -0
  113. data/test/test_log.rb +146 -15
  114. data/test/test_plugin.rb +251 -0
  115. data/test/test_supervisor.rb +65 -57
  116. data/test/test_test_drivers.rb +2 -2
  117. metadata +18 -7
  118. data/lib/fluent/process.rb +0 -504
  119. data/test/test_process.rb +0 -48
@@ -126,7 +126,7 @@ module Fluent
126
126
  merged.configured_in_section = self.configured_in_section || other.configured_in_section
127
127
 
128
128
  merged.argument = other.argument || self.argument
129
- merged.params = other.params.merge(self.params)
129
+ merged.params = self.params.merge(other.params)
130
130
  merged.defaults = self.defaults.merge(other.defaults)
131
131
  merged.sections = {}
132
132
  (self.sections.keys + other.sections.keys).uniq.each do |section_key|
@@ -226,19 +226,41 @@ module Fluent
226
226
  end
227
227
  end
228
228
 
229
+ def config_parameter_option_validate!(name, type, **kwargs, &block)
230
+ if type.nil? && !block
231
+ type = :string
232
+ end
233
+ kwargs.each_key do |key|
234
+ case key
235
+ when :default, :alias, :secret, :skip_accessor, :deprecated, :obsoleted, :desc
236
+ # valid for all types
237
+ when :list
238
+ raise ArgumentError, ":list is valid only for :enum type, but #{type}: #{name}" if type != :enum
239
+ when :value_type
240
+ raise ArgumentError, ":value_type is valid only for :hash and :array, but #{type}: #{name}" if type != :hash && type != :array
241
+ when :symbolize_keys
242
+ raise ArgumentError, ":symbolize_keys is valid only for :hash, but #{type}: #{name}" if type != :hash
243
+ else
244
+ raise ArgumentError, "unknown option '#{key}' for configuration parameter: #{name}"
245
+ end
246
+ end
247
+ end
248
+
229
249
  def parameter_configuration(name, type = nil, **kwargs, &block)
230
- name = name.to_sym
250
+ config_parameter_option_validate!(name, type, **kwargs, &block)
231
251
 
232
- opts = {}
233
- opts[:type] = type
234
- opts.merge!(kwargs)
252
+ name = name.to_sym
235
253
 
236
254
  if block && type
237
255
  raise ArgumentError, "#{name}: both of block and type cannot be specified"
256
+ elsif !block && !type
257
+ type = :string
238
258
  end
259
+ opts = {}
260
+ opts[:type] = type
261
+ opts.merge!(kwargs)
239
262
 
240
263
  begin
241
- type = :string if type.nil?
242
264
  block ||= @type_lookup.call(type)
243
265
  rescue ConfigError
244
266
  # override error message
@@ -252,10 +274,11 @@ module Fluent
252
274
  option_value_type!(name, opts, :deprecated, String)
253
275
  option_value_type!(name, opts, :obsoleted, String)
254
276
  if type == :enum
255
- if !opts.has_key?(:list) || !opts[:list].all?{|v| v.is_a?(Symbol) }
277
+ if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
256
278
  raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
257
279
  end
258
280
  end
281
+ option_value_type!(name, opts, :symbolize_keys, type: :boolean)
259
282
  option_value_type!(name, opts, :value_type, Symbol) # hash, array
260
283
  option_value_type!(name, opts, :skip_accessor, type: :boolean)
261
284
 
@@ -75,6 +75,10 @@ module Fluent
75
75
  @params[key.to_sym]
76
76
  end
77
77
 
78
+ def []=(key, value)
79
+ @params[key.to_sym] = value
80
+ end
81
+
78
82
  def respond_to?(symbol, include_all=false)
79
83
  case symbol
80
84
  when :inspect, :nil?, :to_h, :+, :instance_of?, :kind_of?, :[], :respond_to?, :respond_to_missing?
@@ -64,7 +64,7 @@ module Fluent
64
64
  end
65
65
  end
66
66
 
67
- STRING_TYPE = Proc.new { |val, opts| val }
67
+ STRING_TYPE = Proc.new { |val, opts| val.to_s.encode(Encoding::UTF_8) }
68
68
  ENUM_TYPE = Proc.new { |val, opts|
69
69
  s = val.to_sym
70
70
  list = opts[:list]
@@ -85,7 +85,7 @@ module Fluent
85
85
  value
86
86
  else
87
87
  case type
88
- when :string then value.to_s
88
+ when :string then value.to_s.encode(Encoding::UTF_8)
89
89
  when :integer then value.to_i
90
90
  when :float then value.to_f
91
91
  when :size then Config.size_value(value)
@@ -48,12 +48,8 @@ module Fluent
48
48
  end
49
49
  end
50
50
 
51
- def configure(conf)
52
- @config = conf
53
-
54
- logger = self.respond_to?(:log) ? log : (defined?($log) ? $log : nil)
51
+ def configure_proxy_generate
55
52
  proxy = self.class.merged_configure_proxy
56
- conf.corresponding_proxies << proxy
57
53
 
58
54
  if self.respond_to?(:owner) && self.owner
59
55
  owner_proxy = owner.class.merged_configure_proxy
@@ -63,6 +59,36 @@ module Fluent
63
59
  proxy.overwrite_defaults(owner_proxy) if owner_proxy
64
60
  end
65
61
 
62
+ proxy
63
+ end
64
+
65
+ def configured_section_create(name, conf = nil)
66
+ conf ||= Fluent::Config::Element.new(name.to_s, '', {}, [])
67
+ root_proxy = configure_proxy_generate
68
+ proxy = if name.nil? # root
69
+ root_proxy
70
+ else
71
+ root_proxy.sections[name]
72
+ end
73
+ # take care to raise Fluent::ConfigError if conf mismatched to proxy
74
+ Fluent::Config::SectionGenerator.generate(proxy, conf, nil, nil)
75
+ end
76
+
77
+ def configure(conf)
78
+ @config = conf
79
+
80
+ logger = if self.respond_to?(:log)
81
+ self.log
82
+ elsif self.respond_to?(:owner) && self.owner.respond_to?(:log)
83
+ self.owner.log
84
+ elsif defined?($log)
85
+ $log
86
+ else
87
+ nil
88
+ end
89
+ proxy = configure_proxy_generate
90
+ conf.corresponding_proxies << proxy
91
+
66
92
  # In the nested section, can't get plugin class through proxies so get plugin class here
67
93
  plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
68
94
  root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
@@ -29,13 +29,15 @@ module Fluent
29
29
 
30
30
  def initialize
31
31
  @root_agent = nil
32
- @event_router = nil
33
32
  @default_loop = nil
34
33
  @engine_stopped = false
35
34
 
35
+ @log_event_router = nil
36
36
  @log_emit_thread = nil
37
37
  @log_event_loop_stop = false
38
+ @log_event_loop_graceful_stop = false
38
39
  @log_event_queue = []
40
+ @log_event_verbose = false
39
41
 
40
42
  @suppress_config_dump = false
41
43
 
@@ -58,6 +60,8 @@ module Fluent
58
60
  @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
59
61
  @without_source = system_config.without_source unless system_config.without_source.nil?
60
62
 
63
+ @log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
64
+
61
65
  @root_agent = RootAgent.new(log: log, system_config: @system_config)
62
66
 
63
67
  MessagePackFactory.init
@@ -93,12 +97,12 @@ module Fluent
93
97
  else
94
98
  "section <#{e.name}> is not used in <#{parent_name}>"
95
99
  end
96
- $log.warn message
100
+ $log.warn :worker0, message
97
101
  next
98
102
  end
99
103
  unless e.name == 'system'
100
104
  unless @without_source && e.name == 'source'
101
- $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
105
+ $log.warn :worker0, "parameter '#{key}' in #{e.to_s.strip} is not used."
102
106
  end
103
107
  end
104
108
  }
@@ -107,14 +111,43 @@ module Fluent
107
111
  def configure(conf)
108
112
  # plugins / configuration dumps
109
113
  Gem::Specification.find_all.select{|x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/}.each do |spec|
110
- $log.info "gem '#{spec.name}' version '#{spec.version}'"
114
+ $log.info :worker0, "gem '#{spec.name}' version '#{spec.version}'"
111
115
  end
112
116
 
113
117
  @root_agent.configure(conf)
114
- @event_router = @root_agent.event_router
118
+
119
+ begin
120
+ log_event_agent = @root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
121
+ log_event_router = log_event_agent.event_router
122
+
123
+ # suppress mismatched tags only for <label @FLUENT_LOG> label.
124
+ # it's not suppressed in default event router for non-log-event events
125
+ log_event_router.suppress_missing_match!
126
+
127
+ @log_event_router = log_event_router
128
+
129
+ unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
130
+ unless unmatched_tags.empty?
131
+ $log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
132
+ end
133
+ rescue ArgumentError # ArgumentError "#{label_name} label not found"
134
+ # use default event router if <label @FLUENT_LOG> is missing in configuration
135
+ log_event_router = @root_agent.event_router
136
+
137
+ if Fluent::Log.event_tags.any?{|t| log_event_router.match?(t) }
138
+ @log_event_router = log_event_router
139
+
140
+ unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
141
+ unless unmatched_tags.empty?
142
+ $log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
143
+ end
144
+ end
145
+ end
146
+
147
+ $log.enable_event(true) if @log_event_router
115
148
 
116
149
  unless @suppress_config_dump
117
- $log.info "using configuration file: #{conf.to_s.rstrip}"
150
+ $log.info :worker0, "using configuration file: #{conf.to_s.rstrip}"
118
151
  end
119
152
  end
120
153
 
@@ -148,6 +181,7 @@ module Fluent
148
181
 
149
182
  while sleep(LOG_EMIT_INTERVAL)
150
183
  break if @log_event_loop_stop
184
+ break if @log_event_loop_graceful_stop && @log_event_queue.empty?
151
185
  next if @log_event_queue.empty?
152
186
 
153
187
  # NOTE: thead-safe of slice! depends on GVL
@@ -156,8 +190,9 @@ module Fluent
156
190
 
157
191
  events.each {|tag,time,record|
158
192
  begin
159
- @event_router.emit(tag, time, record)
193
+ @log_event_router.emit(tag, time, record)
160
194
  rescue => e
195
+ # This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
161
196
  $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
162
197
  end
163
198
  }
@@ -165,18 +200,23 @@ module Fluent
165
200
  end
166
201
 
167
202
  def run
203
+ # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
204
+ # so it's (almost) a single worker, worker_id=0
205
+ worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
206
+
168
207
  begin
169
- worker_id = ENV['SERVERENGINE_WORKER_ID']
170
208
  $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
171
209
  start
172
210
 
173
- if @event_router.match?($log.tag)
174
- $log.enable_event
211
+ if @log_event_router
212
+ $log.enable_event(true)
175
213
  @log_emit_thread = Thread.new(&method(:log_event_loop))
214
+ @log_emit_thread.abort_on_exception = true
176
215
  end
177
216
 
178
- $log.info "fluentd worker is now running" # TODO: worker number
217
+ $log.info "fluentd worker is now running", worker: worker_id
179
218
  sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
219
+ $log.info "fluentd worker is now stopping", worker: worker_id
180
220
 
181
221
  rescue Exception => e
182
222
  $log.error "unexpected error", error: e
@@ -184,7 +224,16 @@ module Fluent
184
224
  raise
185
225
  end
186
226
 
187
- $log.info "shutting down fluentd worker" # TODO: worker number
227
+ unless @log_event_verbose
228
+ $log.enable_event(false)
229
+ if @log_emit_thread
230
+ # to make sure to emit all log events into router, before shutting down
231
+ @log_event_loop_graceful_stop = true
232
+ @log_emit_thread.join
233
+ @log_emit_thread = nil
234
+ end
235
+ end
236
+ $log.info "shutting down fluentd worker", worker: worker_id
188
237
  shutdown
189
238
  if @log_emit_thread
190
239
  @log_event_loop_stop = true
@@ -71,6 +71,12 @@ module Fluent
71
71
  attr_reader :pattern_str
72
72
  end
73
73
 
74
+ def suppress_missing_match!
75
+ if @default_collector.respond_to?(:suppress_missing_match!)
76
+ @default_collector.suppress_missing_match!
77
+ end
78
+ end
79
+
74
80
  # called by Agent to add new match pattern and collector
75
81
  def add_rule(pattern, collector)
76
82
  @match_rules << Rule.new(pattern, collector)
@@ -25,7 +25,6 @@ require 'fluent/config'
25
25
  require 'fluent/engine'
26
26
  require 'fluent/rpc'
27
27
  require 'fluent/mixin'
28
- require 'fluent/process'
29
28
  require 'fluent/plugin'
30
29
  require 'fluent/parser'
31
30
  require 'fluent/formatter'
@@ -42,6 +42,14 @@ module Fluent
42
42
 
43
43
  LEVEL_TEXT = %w(trace debug info warn error fatal)
44
44
 
45
+ LOG_EVENT_TAG_PREFIX = 'fluent'
46
+ LOG_EVENT_LABEL = '@FLUENT_LOG'
47
+ LOG_TYPE_SUPERVISOR = :supervisor # only in supervisor, or a worker with --no-supervisor
48
+ LOG_TYPE_WORKER0 = :worker0 # only in a worker with worker_id=0 (without showing worker id)
49
+ LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default)
50
+
51
+ LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
52
+
45
53
  def self.str_to_level(log_level_str)
46
54
  case log_level_str.downcase
47
55
  when "trace" then LEVEL_TRACE
@@ -54,6 +62,10 @@ module Fluent
54
62
  end
55
63
  end
56
64
 
65
+ def self.event_tags
66
+ LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
67
+ end
68
+
57
69
  def initialize(logger, opts={})
58
70
  # overwrites logger.level= so that config reloading resets level of Fluentd::Log
59
71
  orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
@@ -83,8 +95,7 @@ module Fluent
83
95
  @out = logger.instance_variable_get(:@logdev)
84
96
  @level = logger.level + 1
85
97
  @debug_mode = false
86
- @self_event = false
87
- @tag = 'fluent'
98
+ @log_event_enabled = false
88
99
  @time_format = '%Y-%m-%d %H:%M:%S %z '
89
100
  @depth_offset = 1
90
101
  enable_color out.tty?
@@ -97,22 +108,43 @@ module Fluent
97
108
  @optional_attrs = nil
98
109
 
99
110
  @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
111
+
112
+ @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
113
+ @process_type ||= :standalone # to keep behavior of existing code
114
+ case @process_type
115
+ when :supervisor
116
+ @show_supervisor_log = true
117
+ @show_worker0_log = false
118
+ when :worker0
119
+ @show_supervisor_log = false
120
+ @show_worker0_log = true
121
+ when :workers
122
+ @show_supervisor_log = false
123
+ @show_worker0_log = false
124
+ when :standalone
125
+ @show_supervisor_log = true
126
+ @show_worker0_log = true
127
+ else
128
+ raise "BUG: unknown process type for logger:#{@process_type}"
129
+ end
130
+ @worker_id = opts[:worker_id]
131
+ @worker_id_part = "##{@worker_id} " # used only for :default log type in workers
100
132
  end
101
133
 
102
134
  def dup
103
135
  dl_opts = {}
104
136
  dl_opts[:log_level] = @level - 1
105
137
  logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
106
- clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace)
107
- clone.tag = @tag
138
+ clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
108
139
  clone.time_format = @time_format
140
+ clone.log_event_enabled = @log_event_enabled
109
141
  # optional headers/attrs are not copied, because new PluginLogger should have another one of it
110
142
  clone
111
143
  end
112
144
 
145
+ attr_accessor :log_event_enabled
113
146
  attr_accessor :out
114
147
  attr_accessor :level
115
- attr_accessor :tag
116
148
  attr_accessor :time_format
117
149
  attr_accessor :optional_header, :optional_attrs
118
150
 
@@ -134,10 +166,17 @@ module Fluent
134
166
  end
135
167
 
136
168
  def enable_event(b=true)
137
- @self_event = b
169
+ @log_event_enabled = b
138
170
  self
139
171
  end
140
172
 
173
+ # If you want to suppress event emitting in specific thread, please use this method.
174
+ # Events in passed thread are never emitted.
175
+ def disable_events(thread)
176
+ # this method is not symmetric with #enable_event.
177
+ @threads_exclude_events.push(thread) unless @threads_exclude_events.include?(thread)
178
+ end
179
+
141
180
  def enable_color?
142
181
  !@color_reset.empty?
143
182
  end
@@ -163,10 +202,26 @@ module Fluent
163
202
  self
164
203
  end
165
204
 
166
- # If you want to suppress event emitting in specific thread, please use this method.
167
- # Events in passed thread are never emitted.
168
- def disable_events(thread)
169
- @threads_exclude_events.push(thread) unless @threads_exclude_events.include?(thread)
205
+ def log_type(args)
206
+ if LOG_TYPES.include?(args.first)
207
+ args.shift
208
+ else
209
+ LOG_TYPE_DEFAULT
210
+ end
211
+ end
212
+
213
+ # TODO: skip :worker0 logs when Fluentd gracefully restarted
214
+ def skipped_type?(type)
215
+ case type
216
+ when LOG_TYPE_DEFAULT
217
+ false
218
+ when LOG_TYPE_WORKER0
219
+ !@show_worker0_log
220
+ when LOG_TYPE_SUPERVISOR
221
+ !@show_supervisor_log
222
+ else
223
+ raise "BUG: unknown log type:#{type}"
224
+ end
170
225
  end
171
226
 
172
227
  def on_trace(&block)
@@ -176,16 +231,18 @@ module Fluent
176
231
 
177
232
  def trace(*args, &block)
178
233
  return if @level > LEVEL_TRACE
234
+ type = log_type(args)
235
+ return if skipped_type?(type)
179
236
  args << block.call if block
180
237
  time, msg = event(:trace, args)
181
- puts [@color_trace, caller_line(time, @depth_offset, LEVEL_TRACE), msg, @color_reset].join
238
+ puts [@color_trace, caller_line(type, time, @depth_offset, LEVEL_TRACE), msg, @color_reset].join
182
239
  rescue
183
240
  # logger should not raise an exception. This rescue prevents unexpected behaviour.
184
241
  end
185
242
  alias TRACE trace
186
243
 
187
- def trace_backtrace(backtrace=$!.backtrace)
188
- dump_stacktrace(backtrace, LEVEL_TRACE)
244
+ def trace_backtrace(backtrace=$!.backtrace, type: :default)
245
+ dump_stacktrace(type, backtrace, LEVEL_TRACE)
189
246
  end
190
247
 
191
248
  def on_debug(&block)
@@ -195,15 +252,17 @@ module Fluent
195
252
 
196
253
  def debug(*args, &block)
197
254
  return if @level > LEVEL_DEBUG
255
+ type = log_type(args)
256
+ return if skipped_type?(type)
198
257
  args << block.call if block
199
258
  time, msg = event(:debug, args)
200
- puts [@color_debug, caller_line(time, @depth_offset, LEVEL_DEBUG), msg, @color_reset].join
259
+ puts [@color_debug, caller_line(type, time, @depth_offset, LEVEL_DEBUG), msg, @color_reset].join
201
260
  rescue
202
261
  end
203
262
  alias DEBUG debug
204
263
 
205
- def debug_backtrace(backtrace=$!.backtrace)
206
- dump_stacktrace(backtrace, LEVEL_DEBUG)
264
+ def debug_backtrace(backtrace=$!.backtrace, type: :default)
265
+ dump_stacktrace(type, backtrace, LEVEL_DEBUG)
207
266
  end
208
267
 
209
268
  def on_info(&block)
@@ -213,15 +272,17 @@ module Fluent
213
272
 
214
273
  def info(*args, &block)
215
274
  return if @level > LEVEL_INFO
275
+ type = log_type(args)
276
+ return if skipped_type?(type)
216
277
  args << block.call if block
217
278
  time, msg = event(:info, args)
218
- puts [@color_info, caller_line(time, @depth_offset, LEVEL_INFO), msg, @color_reset].join
279
+ puts [@color_info, caller_line(type, time, @depth_offset, LEVEL_INFO), msg, @color_reset].join
219
280
  rescue
220
281
  end
221
282
  alias INFO info
222
283
 
223
- def info_backtrace(backtrace=$!.backtrace)
224
- dump_stacktrace(backtrace, LEVEL_INFO)
284
+ def info_backtrace(backtrace=$!.backtrace, type: :default)
285
+ dump_stacktrace(type, backtrace, LEVEL_INFO)
225
286
  end
226
287
 
227
288
  def on_warn(&block)
@@ -231,15 +292,17 @@ module Fluent
231
292
 
232
293
  def warn(*args, &block)
233
294
  return if @level > LEVEL_WARN
295
+ type = log_type(args)
296
+ return if skipped_type?(type)
234
297
  args << block.call if block
235
298
  time, msg = event(:warn, args)
236
- puts [@color_warn, caller_line(time, @depth_offset, LEVEL_WARN), msg, @color_reset].join
299
+ puts [@color_warn, caller_line(type, time, @depth_offset, LEVEL_WARN), msg, @color_reset].join
237
300
  rescue
238
301
  end
239
302
  alias WARN warn
240
303
 
241
- def warn_backtrace(backtrace=$!.backtrace)
242
- dump_stacktrace(backtrace, LEVEL_WARN)
304
+ def warn_backtrace(backtrace=$!.backtrace, type: :default)
305
+ dump_stacktrace(type, backtrace, LEVEL_WARN)
243
306
  end
244
307
 
245
308
  def on_error(&block)
@@ -249,15 +312,17 @@ module Fluent
249
312
 
250
313
  def error(*args, &block)
251
314
  return if @level > LEVEL_ERROR
315
+ type = log_type(args)
316
+ return if skipped_type?(type)
252
317
  args << block.call if block
253
318
  time, msg = event(:error, args)
254
- puts [@color_error, caller_line(time, @depth_offset, LEVEL_ERROR), msg, @color_reset].join
319
+ puts [@color_error, caller_line(type, time, @depth_offset, LEVEL_ERROR), msg, @color_reset].join
255
320
  rescue
256
321
  end
257
322
  alias ERROR error
258
323
 
259
- def error_backtrace(backtrace=$!.backtrace)
260
- dump_stacktrace(backtrace, LEVEL_ERROR)
324
+ def error_backtrace(backtrace=$!.backtrace, type: :default)
325
+ dump_stacktrace(type, backtrace, LEVEL_ERROR)
261
326
  end
262
327
 
263
328
  def on_fatal(&block)
@@ -267,15 +332,17 @@ module Fluent
267
332
 
268
333
  def fatal(*args, &block)
269
334
  return if @level > LEVEL_FATAL
335
+ type = log_type(args)
336
+ return if skipped_type?(type)
270
337
  args << block.call if block
271
338
  time, msg = event(:fatal, args)
272
- puts [@color_fatal, caller_line(time, @depth_offset, LEVEL_FATAL), msg, @color_reset].join
339
+ puts [@color_fatal, caller_line(type, time, @depth_offset, LEVEL_FATAL), msg, @color_reset].join
273
340
  rescue
274
341
  end
275
342
  alias FATAL fatal
276
343
 
277
- def fatal_backtrace(backtrace=$!.backtrace)
278
- dump_stacktrace(backtrace, LEVEL_FATAL)
344
+ def fatal_backtrace(backtrace=$!.backtrace, type: :default)
345
+ dump_stacktrace(type, backtrace, LEVEL_FATAL)
279
346
  end
280
347
 
281
348
  def puts(msg)
@@ -299,11 +366,11 @@ module Fluent
299
366
  @out.reset if @out.respond_to?(:reset)
300
367
  end
301
368
 
302
- def dump_stacktrace(backtrace, level)
369
+ def dump_stacktrace(type, backtrace, level)
303
370
  return if @level > level
304
371
 
305
372
  time = Time.now
306
- line = caller_line(time, 5, level)
373
+ line = caller_line(type, time, 5, level)
307
374
  if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
308
375
  puts [" ", line, 'suppressed same stacktrace'].join
309
376
  else
@@ -338,20 +405,25 @@ module Fluent
338
405
  end
339
406
  }
340
407
 
341
- unless @threads_exclude_events.include?(Thread.current)
408
+ if @log_event_enabled && !@threads_exclude_events.include?(Thread.current)
342
409
  record = map.dup
343
410
  record.keys.each {|key|
344
411
  record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)
345
412
  }
346
413
  record['message'] = message.dup
347
- @engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
414
+ @engine.push_log_event("#{LOG_EVENT_TAG_PREFIX}.#{level}", Fluent::EventTime.from_time(time), record)
348
415
  end
349
416
 
350
417
  return time, message
351
418
  end
352
419
 
353
- def caller_line(time, depth, level)
354
- log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: "
420
+ def caller_line(type, time, depth, level)
421
+ worker_id_part = if type == :default && (@process_type == :worker0 || @process_type == :workers)
422
+ @worker_id_part
423
+ else
424
+ "".freeze
425
+ end
426
+ log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: #{worker_id_part}"
355
427
  if @debug_mode
356
428
  line = caller(depth+1)[0]
357
429
  if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
@@ -370,7 +442,7 @@ module Fluent
370
442
  # This class enables log_level option in each plugin.
371
443
  #
372
444
  # PluginLogger has same functionality as Log but some methods are forwarded to internal logger
373
- # for keeping logging action consistency in the process, e.g. color, tag, event, etc.
445
+ # for keeping logging action consistency in the process, e.g. color, event, etc.
374
446
  class PluginLogger < Log
375
447
  def initialize(logger)
376
448
  @logger = logger
@@ -396,7 +468,7 @@ module Fluent
396
468
 
397
469
  extend Forwardable
398
470
  def_delegators '@logger', :enable_color?, :enable_debug, :enable_event,
399
- :disable_events, :tag, :tag=, :time_format, :time_format=,
471
+ :disable_events, :log_event_enabled, :log_event_enamed=, :time_format, :time_format=,
400
472
  :event, :caller_line, :puts, :write, :flush, :reset, :out, :out=,
401
473
  :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
402
474
  end
@@ -421,13 +493,17 @@ module Fluent
421
493
  def configure(conf)
422
494
  super
423
495
 
424
- if level = conf['@log_level']
425
- unless @log.is_a?(PluginLogger)
426
- @log = PluginLogger.new($log.dup)
427
- end
428
- @log.level = level
429
- @log.optional_header = "[#{self.class.name}#{plugin_id_configured? ? "(" + @id + ")" : ""}] "
496
+ if plugin_id_configured? || conf['@log_level']
497
+ @log = PluginLogger.new($log.dup) unless @log.is_a?(PluginLogger)
430
498
  @log.optional_attrs = {}
499
+
500
+ if level = conf['@log_level']
501
+ @log.level = level
502
+ end
503
+
504
+ if plugin_id_configured?
505
+ @log.optional_header = "[#{@id}] "
506
+ end
431
507
  end
432
508
  end
433
509