fluentd 0.14.11 → 0.14.12
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +1 -5
- data/ChangeLog +54 -2
- data/example/in_dummy_blocks.conf +17 -0
- data/example/in_forward_tls.conf +14 -0
- data/example/in_forward_workers.conf +21 -0
- data/example/logevents.conf +25 -0
- data/example/out_forward_heartbeat_none.conf +16 -0
- data/example/out_forward_tls.conf +18 -0
- data/example/suppress_config_dump.conf +7 -0
- data/lib/fluent/agent.rb +3 -32
- data/lib/fluent/clock.rb +62 -0
- data/lib/fluent/command/fluentd.rb +12 -0
- data/lib/fluent/compat/input.rb +10 -1
- data/lib/fluent/compat/output.rb +40 -1
- data/lib/fluent/config/configure_proxy.rb +30 -7
- data/lib/fluent/config/section.rb +4 -0
- data/lib/fluent/config/types.rb +2 -2
- data/lib/fluent/configurable.rb +31 -5
- data/lib/fluent/engine.rb +61 -12
- data/lib/fluent/event_router.rb +6 -0
- data/lib/fluent/load.rb +0 -1
- data/lib/fluent/log.rb +118 -42
- data/lib/fluent/match.rb +37 -0
- data/lib/fluent/plugin.rb +25 -3
- data/lib/fluent/plugin/base.rb +4 -0
- data/lib/fluent/plugin/buf_file.rb +38 -14
- data/lib/fluent/plugin/buffer.rb +20 -20
- data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
- data/lib/fluent/plugin/compressable.rb +1 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +3 -6
- data/lib/fluent/plugin/formatter_csv.rb +4 -1
- data/lib/fluent/plugin/formatter_hash.rb +5 -1
- data/lib/fluent/plugin/formatter_json.rb +10 -0
- data/lib/fluent/plugin/formatter_ltsv.rb +2 -1
- data/lib/fluent/plugin/in_dummy.rb +4 -0
- data/lib/fluent/plugin/in_exec.rb +4 -0
- data/lib/fluent/plugin/in_forward.rb +11 -3
- data/lib/fluent/plugin/in_gc_stat.rb +4 -0
- data/lib/fluent/plugin/in_http.rb +4 -0
- data/lib/fluent/plugin/in_monitor_agent.rb +29 -2
- data/lib/fluent/plugin/in_object_space.rb +4 -1
- data/lib/fluent/plugin/in_syslog.rb +4 -0
- data/lib/fluent/plugin/in_tail.rb +193 -116
- data/lib/fluent/plugin/in_tcp.rb +5 -1
- data/lib/fluent/plugin/in_udp.rb +4 -0
- data/lib/fluent/plugin/input.rb +4 -0
- data/lib/fluent/plugin/out_copy.rb +4 -0
- data/lib/fluent/plugin/out_exec.rb +4 -0
- data/lib/fluent/plugin/out_exec_filter.rb +4 -0
- data/lib/fluent/plugin/out_file.rb +70 -30
- data/lib/fluent/plugin/out_forward.rb +132 -28
- data/lib/fluent/plugin/out_null.rb +10 -0
- data/lib/fluent/plugin/out_relabel.rb +4 -0
- data/lib/fluent/plugin/out_roundrobin.rb +4 -0
- data/lib/fluent/plugin/out_secondary_file.rb +5 -0
- data/lib/fluent/plugin/out_stdout.rb +5 -0
- data/lib/fluent/plugin/output.rb +18 -9
- data/lib/fluent/plugin/storage_local.rb +25 -2
- data/lib/fluent/plugin_helper/cert_option.rb +159 -0
- data/lib/fluent/plugin_helper/child_process.rb +6 -6
- data/lib/fluent/plugin_helper/compat_parameters.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +29 -4
- data/lib/fluent/plugin_helper/inject.rb +14 -1
- data/lib/fluent/plugin_helper/server.rb +275 -31
- data/lib/fluent/plugin_helper/socket.rb +144 -4
- data/lib/fluent/plugin_helper/socket_option.rb +2 -17
- data/lib/fluent/plugin_helper/storage.rb +7 -1
- data/lib/fluent/plugin_helper/thread.rb +16 -4
- data/lib/fluent/registry.rb +26 -9
- data/lib/fluent/root_agent.rb +7 -3
- data/lib/fluent/supervisor.rb +37 -15
- data/lib/fluent/system_config.rb +37 -10
- data/lib/fluent/test.rb +2 -0
- data/lib/fluent/test/driver/base.rb +24 -26
- data/lib/fluent/test/helpers.rb +21 -0
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +274 -4
- data/test/config/test_configurable.rb +154 -0
- data/test/config/test_configure_proxy.rb +180 -1
- data/test/config/test_system_config.rb +10 -0
- data/test/config/test_types.rb +1 -0
- data/test/plugin/test_base.rb +4 -0
- data/test/plugin/test_buf_file.rb +241 -9
- data/test/plugin/test_buffer.rb +11 -11
- data/test/plugin/test_buffer_file_chunk.rb +6 -6
- data/test/plugin/test_compressable.rb +3 -0
- data/test/plugin/test_filter.rb +4 -0
- data/test/plugin/test_filter_record_transformer.rb +20 -0
- data/test/plugin/test_formatter_csv.rb +9 -0
- data/test/plugin/test_formatter_hash.rb +35 -0
- data/test/plugin/test_formatter_json.rb +8 -0
- data/test/plugin/test_formatter_ltsv.rb +7 -0
- data/test/plugin/test_in_dummy.rb +7 -3
- data/test/plugin/test_in_monitor_agent.rb +43 -5
- data/test/plugin/test_in_tail.rb +97 -4
- data/test/plugin/test_input.rb +4 -0
- data/test/plugin/test_out_file.rb +46 -7
- data/test/plugin/test_out_forward.rb +59 -7
- data/test/plugin/test_output.rb +10 -4
- data/test/plugin/test_output_as_buffered.rb +37 -25
- data/test/plugin/test_output_as_buffered_compress.rb +1 -1
- data/test/plugin/test_output_as_buffered_retries.rb +6 -6
- data/test/plugin/test_output_as_buffered_secondary.rb +91 -31
- data/test/plugin/test_storage_local.rb +40 -1
- data/test/plugin_helper/test_child_process.rb +29 -28
- data/test/plugin_helper/test_compat_parameters.rb +1 -1
- data/test/plugin_helper/test_inject.rb +27 -9
- data/test/plugin_helper/test_server.rb +822 -50
- data/test/plugin_helper/test_storage.rb +11 -0
- data/test/plugin_helper/test_timer.rb +1 -0
- data/test/test_clock.rb +164 -0
- data/test/test_log.rb +146 -15
- data/test/test_plugin.rb +251 -0
- data/test/test_supervisor.rb +65 -57
- data/test/test_test_drivers.rb +2 -2
- metadata +18 -7
- data/lib/fluent/process.rb +0 -504
- data/test/test_process.rb +0 -48
@@ -126,7 +126,7 @@ module Fluent
|
|
126
126
|
merged.configured_in_section = self.configured_in_section || other.configured_in_section
|
127
127
|
|
128
128
|
merged.argument = other.argument || self.argument
|
129
|
-
merged.params =
|
129
|
+
merged.params = self.params.merge(other.params)
|
130
130
|
merged.defaults = self.defaults.merge(other.defaults)
|
131
131
|
merged.sections = {}
|
132
132
|
(self.sections.keys + other.sections.keys).uniq.each do |section_key|
|
@@ -226,19 +226,41 @@ module Fluent
|
|
226
226
|
end
|
227
227
|
end
|
228
228
|
|
229
|
+
def config_parameter_option_validate!(name, type, **kwargs, &block)
|
230
|
+
if type.nil? && !block
|
231
|
+
type = :string
|
232
|
+
end
|
233
|
+
kwargs.each_key do |key|
|
234
|
+
case key
|
235
|
+
when :default, :alias, :secret, :skip_accessor, :deprecated, :obsoleted, :desc
|
236
|
+
# valid for all types
|
237
|
+
when :list
|
238
|
+
raise ArgumentError, ":list is valid only for :enum type, but #{type}: #{name}" if type != :enum
|
239
|
+
when :value_type
|
240
|
+
raise ArgumentError, ":value_type is valid only for :hash and :array, but #{type}: #{name}" if type != :hash && type != :array
|
241
|
+
when :symbolize_keys
|
242
|
+
raise ArgumentError, ":symbolize_keys is valid only for :hash, but #{type}: #{name}" if type != :hash
|
243
|
+
else
|
244
|
+
raise ArgumentError, "unknown option '#{key}' for configuration parameter: #{name}"
|
245
|
+
end
|
246
|
+
end
|
247
|
+
end
|
248
|
+
|
229
249
|
def parameter_configuration(name, type = nil, **kwargs, &block)
|
230
|
-
name
|
250
|
+
config_parameter_option_validate!(name, type, **kwargs, &block)
|
231
251
|
|
232
|
-
|
233
|
-
opts[:type] = type
|
234
|
-
opts.merge!(kwargs)
|
252
|
+
name = name.to_sym
|
235
253
|
|
236
254
|
if block && type
|
237
255
|
raise ArgumentError, "#{name}: both of block and type cannot be specified"
|
256
|
+
elsif !block && !type
|
257
|
+
type = :string
|
238
258
|
end
|
259
|
+
opts = {}
|
260
|
+
opts[:type] = type
|
261
|
+
opts.merge!(kwargs)
|
239
262
|
|
240
263
|
begin
|
241
|
-
type = :string if type.nil?
|
242
264
|
block ||= @type_lookup.call(type)
|
243
265
|
rescue ConfigError
|
244
266
|
# override error message
|
@@ -252,10 +274,11 @@ module Fluent
|
|
252
274
|
option_value_type!(name, opts, :deprecated, String)
|
253
275
|
option_value_type!(name, opts, :obsoleted, String)
|
254
276
|
if type == :enum
|
255
|
-
if !opts.has_key?(:list) || !opts[:list].all?{|v| v.is_a?(Symbol) }
|
277
|
+
if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
|
256
278
|
raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
|
257
279
|
end
|
258
280
|
end
|
281
|
+
option_value_type!(name, opts, :symbolize_keys, type: :boolean)
|
259
282
|
option_value_type!(name, opts, :value_type, Symbol) # hash, array
|
260
283
|
option_value_type!(name, opts, :skip_accessor, type: :boolean)
|
261
284
|
|
@@ -75,6 +75,10 @@ module Fluent
|
|
75
75
|
@params[key.to_sym]
|
76
76
|
end
|
77
77
|
|
78
|
+
def []=(key, value)
|
79
|
+
@params[key.to_sym] = value
|
80
|
+
end
|
81
|
+
|
78
82
|
def respond_to?(symbol, include_all=false)
|
79
83
|
case symbol
|
80
84
|
when :inspect, :nil?, :to_h, :+, :instance_of?, :kind_of?, :[], :respond_to?, :respond_to_missing?
|
data/lib/fluent/config/types.rb
CHANGED
@@ -64,7 +64,7 @@ module Fluent
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
-
STRING_TYPE = Proc.new { |val, opts| val }
|
67
|
+
STRING_TYPE = Proc.new { |val, opts| val.to_s.encode(Encoding::UTF_8) }
|
68
68
|
ENUM_TYPE = Proc.new { |val, opts|
|
69
69
|
s = val.to_sym
|
70
70
|
list = opts[:list]
|
@@ -85,7 +85,7 @@ module Fluent
|
|
85
85
|
value
|
86
86
|
else
|
87
87
|
case type
|
88
|
-
when :string then value.to_s
|
88
|
+
when :string then value.to_s.encode(Encoding::UTF_8)
|
89
89
|
when :integer then value.to_i
|
90
90
|
when :float then value.to_f
|
91
91
|
when :size then Config.size_value(value)
|
data/lib/fluent/configurable.rb
CHANGED
@@ -48,12 +48,8 @@ module Fluent
|
|
48
48
|
end
|
49
49
|
end
|
50
50
|
|
51
|
-
def
|
52
|
-
@config = conf
|
53
|
-
|
54
|
-
logger = self.respond_to?(:log) ? log : (defined?($log) ? $log : nil)
|
51
|
+
def configure_proxy_generate
|
55
52
|
proxy = self.class.merged_configure_proxy
|
56
|
-
conf.corresponding_proxies << proxy
|
57
53
|
|
58
54
|
if self.respond_to?(:owner) && self.owner
|
59
55
|
owner_proxy = owner.class.merged_configure_proxy
|
@@ -63,6 +59,36 @@ module Fluent
|
|
63
59
|
proxy.overwrite_defaults(owner_proxy) if owner_proxy
|
64
60
|
end
|
65
61
|
|
62
|
+
proxy
|
63
|
+
end
|
64
|
+
|
65
|
+
def configured_section_create(name, conf = nil)
|
66
|
+
conf ||= Fluent::Config::Element.new(name.to_s, '', {}, [])
|
67
|
+
root_proxy = configure_proxy_generate
|
68
|
+
proxy = if name.nil? # root
|
69
|
+
root_proxy
|
70
|
+
else
|
71
|
+
root_proxy.sections[name]
|
72
|
+
end
|
73
|
+
# take care to raise Fluent::ConfigError if conf mismatched to proxy
|
74
|
+
Fluent::Config::SectionGenerator.generate(proxy, conf, nil, nil)
|
75
|
+
end
|
76
|
+
|
77
|
+
def configure(conf)
|
78
|
+
@config = conf
|
79
|
+
|
80
|
+
logger = if self.respond_to?(:log)
|
81
|
+
self.log
|
82
|
+
elsif self.respond_to?(:owner) && self.owner.respond_to?(:log)
|
83
|
+
self.owner.log
|
84
|
+
elsif defined?($log)
|
85
|
+
$log
|
86
|
+
else
|
87
|
+
nil
|
88
|
+
end
|
89
|
+
proxy = configure_proxy_generate
|
90
|
+
conf.corresponding_proxies << proxy
|
91
|
+
|
66
92
|
# In the nested section, can't get plugin class through proxies so get plugin class here
|
67
93
|
plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
|
68
94
|
root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
|
data/lib/fluent/engine.rb
CHANGED
@@ -29,13 +29,15 @@ module Fluent
|
|
29
29
|
|
30
30
|
def initialize
|
31
31
|
@root_agent = nil
|
32
|
-
@event_router = nil
|
33
32
|
@default_loop = nil
|
34
33
|
@engine_stopped = false
|
35
34
|
|
35
|
+
@log_event_router = nil
|
36
36
|
@log_emit_thread = nil
|
37
37
|
@log_event_loop_stop = false
|
38
|
+
@log_event_loop_graceful_stop = false
|
38
39
|
@log_event_queue = []
|
40
|
+
@log_event_verbose = false
|
39
41
|
|
40
42
|
@suppress_config_dump = false
|
41
43
|
|
@@ -58,6 +60,8 @@ module Fluent
|
|
58
60
|
@suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
|
59
61
|
@without_source = system_config.without_source unless system_config.without_source.nil?
|
60
62
|
|
63
|
+
@log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
|
64
|
+
|
61
65
|
@root_agent = RootAgent.new(log: log, system_config: @system_config)
|
62
66
|
|
63
67
|
MessagePackFactory.init
|
@@ -93,12 +97,12 @@ module Fluent
|
|
93
97
|
else
|
94
98
|
"section <#{e.name}> is not used in <#{parent_name}>"
|
95
99
|
end
|
96
|
-
$log.warn message
|
100
|
+
$log.warn :worker0, message
|
97
101
|
next
|
98
102
|
end
|
99
103
|
unless e.name == 'system'
|
100
104
|
unless @without_source && e.name == 'source'
|
101
|
-
$log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
|
105
|
+
$log.warn :worker0, "parameter '#{key}' in #{e.to_s.strip} is not used."
|
102
106
|
end
|
103
107
|
end
|
104
108
|
}
|
@@ -107,14 +111,43 @@ module Fluent
|
|
107
111
|
def configure(conf)
|
108
112
|
# plugins / configuration dumps
|
109
113
|
Gem::Specification.find_all.select{|x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/}.each do |spec|
|
110
|
-
$log.info "gem '#{spec.name}' version '#{spec.version}'"
|
114
|
+
$log.info :worker0, "gem '#{spec.name}' version '#{spec.version}'"
|
111
115
|
end
|
112
116
|
|
113
117
|
@root_agent.configure(conf)
|
114
|
-
|
118
|
+
|
119
|
+
begin
|
120
|
+
log_event_agent = @root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
|
121
|
+
log_event_router = log_event_agent.event_router
|
122
|
+
|
123
|
+
# suppress mismatched tags only for <label @FLUENT_LOG> label.
|
124
|
+
# it's not suppressed in default event router for non-log-event events
|
125
|
+
log_event_router.suppress_missing_match!
|
126
|
+
|
127
|
+
@log_event_router = log_event_router
|
128
|
+
|
129
|
+
unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
|
130
|
+
unless unmatched_tags.empty?
|
131
|
+
$log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
|
132
|
+
end
|
133
|
+
rescue ArgumentError # ArgumentError "#{label_name} label not found"
|
134
|
+
# use default event router if <label @FLUENT_LOG> is missing in configuration
|
135
|
+
log_event_router = @root_agent.event_router
|
136
|
+
|
137
|
+
if Fluent::Log.event_tags.any?{|t| log_event_router.match?(t) }
|
138
|
+
@log_event_router = log_event_router
|
139
|
+
|
140
|
+
unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
|
141
|
+
unless unmatched_tags.empty?
|
142
|
+
$log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
$log.enable_event(true) if @log_event_router
|
115
148
|
|
116
149
|
unless @suppress_config_dump
|
117
|
-
$log.info "using configuration file: #{conf.to_s.rstrip}"
|
150
|
+
$log.info :worker0, "using configuration file: #{conf.to_s.rstrip}"
|
118
151
|
end
|
119
152
|
end
|
120
153
|
|
@@ -148,6 +181,7 @@ module Fluent
|
|
148
181
|
|
149
182
|
while sleep(LOG_EMIT_INTERVAL)
|
150
183
|
break if @log_event_loop_stop
|
184
|
+
break if @log_event_loop_graceful_stop && @log_event_queue.empty?
|
151
185
|
next if @log_event_queue.empty?
|
152
186
|
|
153
187
|
# NOTE: thead-safe of slice! depends on GVL
|
@@ -156,8 +190,9 @@ module Fluent
|
|
156
190
|
|
157
191
|
events.each {|tag,time,record|
|
158
192
|
begin
|
159
|
-
@
|
193
|
+
@log_event_router.emit(tag, time, record)
|
160
194
|
rescue => e
|
195
|
+
# This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
|
161
196
|
$log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
|
162
197
|
end
|
163
198
|
}
|
@@ -165,18 +200,23 @@ module Fluent
|
|
165
200
|
end
|
166
201
|
|
167
202
|
def run
|
203
|
+
# if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
|
204
|
+
# so it's (almost) a single worker, worker_id=0
|
205
|
+
worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
|
206
|
+
|
168
207
|
begin
|
169
|
-
worker_id = ENV['SERVERENGINE_WORKER_ID']
|
170
208
|
$log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
|
171
209
|
start
|
172
210
|
|
173
|
-
if @
|
174
|
-
$log.enable_event
|
211
|
+
if @log_event_router
|
212
|
+
$log.enable_event(true)
|
175
213
|
@log_emit_thread = Thread.new(&method(:log_event_loop))
|
214
|
+
@log_emit_thread.abort_on_exception = true
|
176
215
|
end
|
177
216
|
|
178
|
-
$log.info "fluentd worker is now running"
|
217
|
+
$log.info "fluentd worker is now running", worker: worker_id
|
179
218
|
sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
|
219
|
+
$log.info "fluentd worker is now stopping", worker: worker_id
|
180
220
|
|
181
221
|
rescue Exception => e
|
182
222
|
$log.error "unexpected error", error: e
|
@@ -184,7 +224,16 @@ module Fluent
|
|
184
224
|
raise
|
185
225
|
end
|
186
226
|
|
187
|
-
|
227
|
+
unless @log_event_verbose
|
228
|
+
$log.enable_event(false)
|
229
|
+
if @log_emit_thread
|
230
|
+
# to make sure to emit all log events into router, before shutting down
|
231
|
+
@log_event_loop_graceful_stop = true
|
232
|
+
@log_emit_thread.join
|
233
|
+
@log_emit_thread = nil
|
234
|
+
end
|
235
|
+
end
|
236
|
+
$log.info "shutting down fluentd worker", worker: worker_id
|
188
237
|
shutdown
|
189
238
|
if @log_emit_thread
|
190
239
|
@log_event_loop_stop = true
|
data/lib/fluent/event_router.rb
CHANGED
@@ -71,6 +71,12 @@ module Fluent
|
|
71
71
|
attr_reader :pattern_str
|
72
72
|
end
|
73
73
|
|
74
|
+
def suppress_missing_match!
|
75
|
+
if @default_collector.respond_to?(:suppress_missing_match!)
|
76
|
+
@default_collector.suppress_missing_match!
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
74
80
|
# called by Agent to add new match pattern and collector
|
75
81
|
def add_rule(pattern, collector)
|
76
82
|
@match_rules << Rule.new(pattern, collector)
|
data/lib/fluent/load.rb
CHANGED
data/lib/fluent/log.rb
CHANGED
@@ -42,6 +42,14 @@ module Fluent
|
|
42
42
|
|
43
43
|
LEVEL_TEXT = %w(trace debug info warn error fatal)
|
44
44
|
|
45
|
+
LOG_EVENT_TAG_PREFIX = 'fluent'
|
46
|
+
LOG_EVENT_LABEL = '@FLUENT_LOG'
|
47
|
+
LOG_TYPE_SUPERVISOR = :supervisor # only in supervisor, or a worker with --no-supervisor
|
48
|
+
LOG_TYPE_WORKER0 = :worker0 # only in a worker with worker_id=0 (without showing worker id)
|
49
|
+
LOG_TYPE_DEFAULT = :default # show logs in all supervisor/workers, with worker id in workers (default)
|
50
|
+
|
51
|
+
LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
|
52
|
+
|
45
53
|
def self.str_to_level(log_level_str)
|
46
54
|
case log_level_str.downcase
|
47
55
|
when "trace" then LEVEL_TRACE
|
@@ -54,6 +62,10 @@ module Fluent
|
|
54
62
|
end
|
55
63
|
end
|
56
64
|
|
65
|
+
def self.event_tags
|
66
|
+
LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
|
67
|
+
end
|
68
|
+
|
57
69
|
def initialize(logger, opts={})
|
58
70
|
# overwrites logger.level= so that config reloading resets level of Fluentd::Log
|
59
71
|
orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
|
@@ -83,8 +95,7 @@ module Fluent
|
|
83
95
|
@out = logger.instance_variable_get(:@logdev)
|
84
96
|
@level = logger.level + 1
|
85
97
|
@debug_mode = false
|
86
|
-
@
|
87
|
-
@tag = 'fluent'
|
98
|
+
@log_event_enabled = false
|
88
99
|
@time_format = '%Y-%m-%d %H:%M:%S %z '
|
89
100
|
@depth_offset = 1
|
90
101
|
enable_color out.tty?
|
@@ -97,22 +108,43 @@ module Fluent
|
|
97
108
|
@optional_attrs = nil
|
98
109
|
|
99
110
|
@suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
|
111
|
+
|
112
|
+
@process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
|
113
|
+
@process_type ||= :standalone # to keep behavior of existing code
|
114
|
+
case @process_type
|
115
|
+
when :supervisor
|
116
|
+
@show_supervisor_log = true
|
117
|
+
@show_worker0_log = false
|
118
|
+
when :worker0
|
119
|
+
@show_supervisor_log = false
|
120
|
+
@show_worker0_log = true
|
121
|
+
when :workers
|
122
|
+
@show_supervisor_log = false
|
123
|
+
@show_worker0_log = false
|
124
|
+
when :standalone
|
125
|
+
@show_supervisor_log = true
|
126
|
+
@show_worker0_log = true
|
127
|
+
else
|
128
|
+
raise "BUG: unknown process type for logger:#{@process_type}"
|
129
|
+
end
|
130
|
+
@worker_id = opts[:worker_id]
|
131
|
+
@worker_id_part = "##{@worker_id} " # used only for :default log type in workers
|
100
132
|
end
|
101
133
|
|
102
134
|
def dup
|
103
135
|
dl_opts = {}
|
104
136
|
dl_opts[:log_level] = @level - 1
|
105
137
|
logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
|
106
|
-
clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace)
|
107
|
-
clone.tag = @tag
|
138
|
+
clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type, worker_id: @worker_id)
|
108
139
|
clone.time_format = @time_format
|
140
|
+
clone.log_event_enabled = @log_event_enabled
|
109
141
|
# optional headers/attrs are not copied, because new PluginLogger should have another one of it
|
110
142
|
clone
|
111
143
|
end
|
112
144
|
|
145
|
+
attr_accessor :log_event_enabled
|
113
146
|
attr_accessor :out
|
114
147
|
attr_accessor :level
|
115
|
-
attr_accessor :tag
|
116
148
|
attr_accessor :time_format
|
117
149
|
attr_accessor :optional_header, :optional_attrs
|
118
150
|
|
@@ -134,10 +166,17 @@ module Fluent
|
|
134
166
|
end
|
135
167
|
|
136
168
|
def enable_event(b=true)
|
137
|
-
@
|
169
|
+
@log_event_enabled = b
|
138
170
|
self
|
139
171
|
end
|
140
172
|
|
173
|
+
# If you want to suppress event emitting in specific thread, please use this method.
|
174
|
+
# Events in passed thread are never emitted.
|
175
|
+
def disable_events(thread)
|
176
|
+
# this method is not symmetric with #enable_event.
|
177
|
+
@threads_exclude_events.push(thread) unless @threads_exclude_events.include?(thread)
|
178
|
+
end
|
179
|
+
|
141
180
|
def enable_color?
|
142
181
|
!@color_reset.empty?
|
143
182
|
end
|
@@ -163,10 +202,26 @@ module Fluent
|
|
163
202
|
self
|
164
203
|
end
|
165
204
|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
205
|
+
def log_type(args)
|
206
|
+
if LOG_TYPES.include?(args.first)
|
207
|
+
args.shift
|
208
|
+
else
|
209
|
+
LOG_TYPE_DEFAULT
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# TODO: skip :worker0 logs when Fluentd gracefully restarted
|
214
|
+
def skipped_type?(type)
|
215
|
+
case type
|
216
|
+
when LOG_TYPE_DEFAULT
|
217
|
+
false
|
218
|
+
when LOG_TYPE_WORKER0
|
219
|
+
!@show_worker0_log
|
220
|
+
when LOG_TYPE_SUPERVISOR
|
221
|
+
!@show_supervisor_log
|
222
|
+
else
|
223
|
+
raise "BUG: unknown log type:#{type}"
|
224
|
+
end
|
170
225
|
end
|
171
226
|
|
172
227
|
def on_trace(&block)
|
@@ -176,16 +231,18 @@ module Fluent
|
|
176
231
|
|
177
232
|
def trace(*args, &block)
|
178
233
|
return if @level > LEVEL_TRACE
|
234
|
+
type = log_type(args)
|
235
|
+
return if skipped_type?(type)
|
179
236
|
args << block.call if block
|
180
237
|
time, msg = event(:trace, args)
|
181
|
-
puts [@color_trace, caller_line(time, @depth_offset, LEVEL_TRACE), msg, @color_reset].join
|
238
|
+
puts [@color_trace, caller_line(type, time, @depth_offset, LEVEL_TRACE), msg, @color_reset].join
|
182
239
|
rescue
|
183
240
|
# logger should not raise an exception. This rescue prevents unexpected behaviour.
|
184
241
|
end
|
185
242
|
alias TRACE trace
|
186
243
|
|
187
|
-
def trace_backtrace(backtrace=$!.backtrace)
|
188
|
-
dump_stacktrace(backtrace, LEVEL_TRACE)
|
244
|
+
def trace_backtrace(backtrace=$!.backtrace, type: :default)
|
245
|
+
dump_stacktrace(type, backtrace, LEVEL_TRACE)
|
189
246
|
end
|
190
247
|
|
191
248
|
def on_debug(&block)
|
@@ -195,15 +252,17 @@ module Fluent
|
|
195
252
|
|
196
253
|
def debug(*args, &block)
|
197
254
|
return if @level > LEVEL_DEBUG
|
255
|
+
type = log_type(args)
|
256
|
+
return if skipped_type?(type)
|
198
257
|
args << block.call if block
|
199
258
|
time, msg = event(:debug, args)
|
200
|
-
puts [@color_debug, caller_line(time, @depth_offset, LEVEL_DEBUG), msg, @color_reset].join
|
259
|
+
puts [@color_debug, caller_line(type, time, @depth_offset, LEVEL_DEBUG), msg, @color_reset].join
|
201
260
|
rescue
|
202
261
|
end
|
203
262
|
alias DEBUG debug
|
204
263
|
|
205
|
-
def debug_backtrace(backtrace=$!.backtrace)
|
206
|
-
dump_stacktrace(backtrace, LEVEL_DEBUG)
|
264
|
+
def debug_backtrace(backtrace=$!.backtrace, type: :default)
|
265
|
+
dump_stacktrace(type, backtrace, LEVEL_DEBUG)
|
207
266
|
end
|
208
267
|
|
209
268
|
def on_info(&block)
|
@@ -213,15 +272,17 @@ module Fluent
|
|
213
272
|
|
214
273
|
def info(*args, &block)
|
215
274
|
return if @level > LEVEL_INFO
|
275
|
+
type = log_type(args)
|
276
|
+
return if skipped_type?(type)
|
216
277
|
args << block.call if block
|
217
278
|
time, msg = event(:info, args)
|
218
|
-
puts [@color_info, caller_line(time, @depth_offset, LEVEL_INFO), msg, @color_reset].join
|
279
|
+
puts [@color_info, caller_line(type, time, @depth_offset, LEVEL_INFO), msg, @color_reset].join
|
219
280
|
rescue
|
220
281
|
end
|
221
282
|
alias INFO info
|
222
283
|
|
223
|
-
def info_backtrace(backtrace=$!.backtrace)
|
224
|
-
dump_stacktrace(backtrace, LEVEL_INFO)
|
284
|
+
def info_backtrace(backtrace=$!.backtrace, type: :default)
|
285
|
+
dump_stacktrace(type, backtrace, LEVEL_INFO)
|
225
286
|
end
|
226
287
|
|
227
288
|
def on_warn(&block)
|
@@ -231,15 +292,17 @@ module Fluent
|
|
231
292
|
|
232
293
|
def warn(*args, &block)
|
233
294
|
return if @level > LEVEL_WARN
|
295
|
+
type = log_type(args)
|
296
|
+
return if skipped_type?(type)
|
234
297
|
args << block.call if block
|
235
298
|
time, msg = event(:warn, args)
|
236
|
-
puts [@color_warn, caller_line(time, @depth_offset, LEVEL_WARN), msg, @color_reset].join
|
299
|
+
puts [@color_warn, caller_line(type, time, @depth_offset, LEVEL_WARN), msg, @color_reset].join
|
237
300
|
rescue
|
238
301
|
end
|
239
302
|
alias WARN warn
|
240
303
|
|
241
|
-
def warn_backtrace(backtrace=$!.backtrace)
|
242
|
-
dump_stacktrace(backtrace, LEVEL_WARN)
|
304
|
+
def warn_backtrace(backtrace=$!.backtrace, type: :default)
|
305
|
+
dump_stacktrace(type, backtrace, LEVEL_WARN)
|
243
306
|
end
|
244
307
|
|
245
308
|
def on_error(&block)
|
@@ -249,15 +312,17 @@ module Fluent
|
|
249
312
|
|
250
313
|
def error(*args, &block)
|
251
314
|
return if @level > LEVEL_ERROR
|
315
|
+
type = log_type(args)
|
316
|
+
return if skipped_type?(type)
|
252
317
|
args << block.call if block
|
253
318
|
time, msg = event(:error, args)
|
254
|
-
puts [@color_error, caller_line(time, @depth_offset, LEVEL_ERROR), msg, @color_reset].join
|
319
|
+
puts [@color_error, caller_line(type, time, @depth_offset, LEVEL_ERROR), msg, @color_reset].join
|
255
320
|
rescue
|
256
321
|
end
|
257
322
|
alias ERROR error
|
258
323
|
|
259
|
-
def error_backtrace(backtrace=$!.backtrace)
|
260
|
-
dump_stacktrace(backtrace, LEVEL_ERROR)
|
324
|
+
def error_backtrace(backtrace=$!.backtrace, type: :default)
|
325
|
+
dump_stacktrace(type, backtrace, LEVEL_ERROR)
|
261
326
|
end
|
262
327
|
|
263
328
|
def on_fatal(&block)
|
@@ -267,15 +332,17 @@ module Fluent
|
|
267
332
|
|
268
333
|
def fatal(*args, &block)
|
269
334
|
return if @level > LEVEL_FATAL
|
335
|
+
type = log_type(args)
|
336
|
+
return if skipped_type?(type)
|
270
337
|
args << block.call if block
|
271
338
|
time, msg = event(:fatal, args)
|
272
|
-
puts [@color_fatal, caller_line(time, @depth_offset, LEVEL_FATAL), msg, @color_reset].join
|
339
|
+
puts [@color_fatal, caller_line(type, time, @depth_offset, LEVEL_FATAL), msg, @color_reset].join
|
273
340
|
rescue
|
274
341
|
end
|
275
342
|
alias FATAL fatal
|
276
343
|
|
277
|
-
def fatal_backtrace(backtrace=$!.backtrace)
|
278
|
-
dump_stacktrace(backtrace, LEVEL_FATAL)
|
344
|
+
def fatal_backtrace(backtrace=$!.backtrace, type: :default)
|
345
|
+
dump_stacktrace(type, backtrace, LEVEL_FATAL)
|
279
346
|
end
|
280
347
|
|
281
348
|
def puts(msg)
|
@@ -299,11 +366,11 @@ module Fluent
|
|
299
366
|
@out.reset if @out.respond_to?(:reset)
|
300
367
|
end
|
301
368
|
|
302
|
-
def dump_stacktrace(backtrace, level)
|
369
|
+
def dump_stacktrace(type, backtrace, level)
|
303
370
|
return if @level > level
|
304
371
|
|
305
372
|
time = Time.now
|
306
|
-
line = caller_line(time, 5, level)
|
373
|
+
line = caller_line(type, time, 5, level)
|
307
374
|
if @suppress_repeated_stacktrace && (Thread.current[:last_repeated_stacktrace] == backtrace)
|
308
375
|
puts [" ", line, 'suppressed same stacktrace'].join
|
309
376
|
else
|
@@ -338,20 +405,25 @@ module Fluent
|
|
338
405
|
end
|
339
406
|
}
|
340
407
|
|
341
|
-
|
408
|
+
if @log_event_enabled && !@threads_exclude_events.include?(Thread.current)
|
342
409
|
record = map.dup
|
343
410
|
record.keys.each {|key|
|
344
411
|
record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)
|
345
412
|
}
|
346
413
|
record['message'] = message.dup
|
347
|
-
@engine.push_log_event("#{
|
414
|
+
@engine.push_log_event("#{LOG_EVENT_TAG_PREFIX}.#{level}", Fluent::EventTime.from_time(time), record)
|
348
415
|
end
|
349
416
|
|
350
417
|
return time, message
|
351
418
|
end
|
352
419
|
|
353
|
-
def caller_line(time, depth, level)
|
354
|
-
|
420
|
+
def caller_line(type, time, depth, level)
|
421
|
+
worker_id_part = if type == :default && (@process_type == :worker0 || @process_type == :workers)
|
422
|
+
@worker_id_part
|
423
|
+
else
|
424
|
+
"".freeze
|
425
|
+
end
|
426
|
+
log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: #{worker_id_part}"
|
355
427
|
if @debug_mode
|
356
428
|
line = caller(depth+1)[0]
|
357
429
|
if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
|
@@ -370,7 +442,7 @@ module Fluent
|
|
370
442
|
# This class enables log_level option in each plugin.
|
371
443
|
#
|
372
444
|
# PluginLogger has same functionality as Log but some methods are forwarded to internal logger
|
373
|
-
# for keeping logging action consistency in the process, e.g. color,
|
445
|
+
# for keeping logging action consistency in the process, e.g. color, event, etc.
|
374
446
|
class PluginLogger < Log
|
375
447
|
def initialize(logger)
|
376
448
|
@logger = logger
|
@@ -396,7 +468,7 @@ module Fluent
|
|
396
468
|
|
397
469
|
extend Forwardable
|
398
470
|
def_delegators '@logger', :enable_color?, :enable_debug, :enable_event,
|
399
|
-
:disable_events, :
|
471
|
+
:disable_events, :log_event_enabled, :log_event_enamed=, :time_format, :time_format=,
|
400
472
|
:event, :caller_line, :puts, :write, :flush, :reset, :out, :out=,
|
401
473
|
:optional_header, :optional_header=, :optional_attrs, :optional_attrs=
|
402
474
|
end
|
@@ -421,13 +493,17 @@ module Fluent
|
|
421
493
|
def configure(conf)
|
422
494
|
super
|
423
495
|
|
424
|
-
if
|
425
|
-
unless @log.is_a?(PluginLogger)
|
426
|
-
@log = PluginLogger.new($log.dup)
|
427
|
-
end
|
428
|
-
@log.level = level
|
429
|
-
@log.optional_header = "[#{self.class.name}#{plugin_id_configured? ? "(" + @id + ")" : ""}] "
|
496
|
+
if plugin_id_configured? || conf['@log_level']
|
497
|
+
@log = PluginLogger.new($log.dup) unless @log.is_a?(PluginLogger)
|
430
498
|
@log.optional_attrs = {}
|
499
|
+
|
500
|
+
if level = conf['@log_level']
|
501
|
+
@log.level = level
|
502
|
+
end
|
503
|
+
|
504
|
+
if plugin_id_configured?
|
505
|
+
@log.optional_header = "[#{@id}] "
|
506
|
+
end
|
431
507
|
end
|
432
508
|
end
|
433
509
|
|