fluentd 0.10.35 → 0.10.36
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- data/.travis.yml +13 -0
- data/ChangeLog +9 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/buffer.rb +210 -214
- data/lib/fluent/command/fluentd.rb +4 -0
- data/lib/fluent/config.rb +1 -0
- data/lib/fluent/engine.rb +10 -10
- data/lib/fluent/output.rb +404 -406
- data/lib/fluent/plugin/buf_file.rb +146 -151
- data/lib/fluent/plugin/buf_memory.rb +62 -67
- data/lib/fluent/plugin/in_debug_agent.rb +27 -31
- data/lib/fluent/plugin/in_exec.rb +86 -90
- data/lib/fluent/plugin/in_forward.rb +171 -171
- data/lib/fluent/plugin/in_gc_stat.rb +43 -47
- data/lib/fluent/plugin/in_http.rb +214 -216
- data/lib/fluent/plugin/in_monitor_agent.rb +212 -214
- data/lib/fluent/plugin/in_object_space.rb +75 -79
- data/lib/fluent/plugin/in_status.rb +44 -50
- data/lib/fluent/plugin/in_stream.rb +159 -160
- data/lib/fluent/plugin/in_syslog.rb +149 -153
- data/lib/fluent/plugin/in_tail.rb +382 -387
- data/lib/fluent/plugin/out_copy.rb +40 -45
- data/lib/fluent/plugin/out_exec.rb +52 -57
- data/lib/fluent/plugin/out_exec_filter.rb +327 -331
- data/lib/fluent/plugin/out_file.rb +78 -74
- data/lib/fluent/plugin/out_forward.rb +410 -414
- data/lib/fluent/plugin/out_null.rb +15 -19
- data/lib/fluent/plugin/out_roundrobin.rb +63 -68
- data/lib/fluent/plugin/out_stdout.rb +9 -14
- data/lib/fluent/plugin/out_stream.rb +83 -90
- data/lib/fluent/plugin/out_test.rb +42 -46
- data/lib/fluent/supervisor.rb +15 -0
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/in_stream.rb +2 -0
- data/test/plugin/out_file.rb +19 -1
- metadata +6 -5
@@ -16,273 +16,271 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
+
class MonitorAgentInput < Input
|
20
|
+
Plugin.register_input('monitor_agent', self)
|
19
21
|
|
20
|
-
|
21
|
-
Plugin.register_input('monitor_agent', self)
|
22
|
+
require 'webrick'
|
22
23
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
require 'cgi'
|
27
|
-
super
|
28
|
-
end
|
29
|
-
|
30
|
-
config_param :bind, :string, :default => '0.0.0.0'
|
31
|
-
config_param :port, :integer, :default => 24220
|
32
|
-
|
33
|
-
class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
|
34
|
-
def initialize(server, agent)
|
35
|
-
@agent = agent
|
24
|
+
def initialize
|
25
|
+
require 'cgi'
|
26
|
+
super
|
36
27
|
end
|
37
28
|
|
38
|
-
|
39
|
-
|
40
|
-
code, header, body = process(req, res)
|
41
|
-
rescue
|
42
|
-
code, header, body = render_json_error(500, {
|
43
|
-
'message '=> 'Internal Server Error',
|
44
|
-
'error' => "#{$!}",
|
45
|
-
'backgrace'=> $!.backtrace,
|
46
|
-
})
|
47
|
-
end
|
29
|
+
config_param :bind, :string, :default => '0.0.0.0'
|
30
|
+
config_param :port, :integer, :default => 24220
|
48
31
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
res[k] = v
|
53
|
-
}
|
54
|
-
res.body = body
|
55
|
-
end
|
56
|
-
|
57
|
-
def build_object(req, res)
|
58
|
-
unless req.path_info == ""
|
59
|
-
return render_json_error(404, "Not found")
|
32
|
+
class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
|
33
|
+
def initialize(server, agent)
|
34
|
+
@agent = agent
|
60
35
|
end
|
61
36
|
|
62
|
-
|
63
|
-
if req.query_string
|
37
|
+
def do_GET(req, res)
|
64
38
|
begin
|
65
|
-
|
39
|
+
code, header, body = process(req, res)
|
66
40
|
rescue
|
67
|
-
|
41
|
+
code, header, body = render_json_error(500, {
|
42
|
+
'message '=> 'Internal Server Error',
|
43
|
+
'error' => "#{$!}",
|
44
|
+
'backgrace'=> $!.backtrace,
|
45
|
+
})
|
68
46
|
end
|
69
|
-
else
|
70
|
-
qs = Hash.new {|h,k| [] }
|
71
|
-
end
|
72
47
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
48
|
+
# set response code, header and body
|
49
|
+
res.status = code
|
50
|
+
header.each_pair {|k,v|
|
51
|
+
res[k] = v
|
52
|
+
}
|
53
|
+
res.body = body
|
79
54
|
end
|
80
55
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
list = [obj]
|
85
|
-
else
|
86
|
-
list = []
|
56
|
+
def build_object(req, res)
|
57
|
+
unless req.path_info == ""
|
58
|
+
return render_json_error(404, "Not found")
|
87
59
|
end
|
88
60
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
61
|
+
# parse ?=query string
|
62
|
+
if req.query_string
|
63
|
+
begin
|
64
|
+
qs = CGI.parse(req.query_string)
|
65
|
+
rescue
|
66
|
+
return render_json_error(400, "Invalid query string")
|
67
|
+
end
|
93
68
|
else
|
94
|
-
|
69
|
+
qs = Hash.new {|h,k| [] }
|
95
70
|
end
|
96
71
|
|
97
|
-
|
98
|
-
#
|
99
|
-
|
72
|
+
# if ?debug=1 is set, set :with_debug_info for get_monitor_info
|
73
|
+
# and :pretty_json for render_json_error
|
74
|
+
opts = {}
|
75
|
+
if s = qs['debug'] and s[0]
|
76
|
+
opts[:with_debug_info] = true
|
77
|
+
opts[:pretty_json] = true
|
78
|
+
end
|
100
79
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
80
|
+
if tags = qs['tag'] and tag = tags[0]
|
81
|
+
# ?tag= to search an output plugin by match pattern
|
82
|
+
if obj = @agent.plugin_info_by_tag(tag, opts)
|
83
|
+
list = [obj]
|
84
|
+
else
|
85
|
+
list = []
|
86
|
+
end
|
105
87
|
|
106
|
-
|
107
|
-
|
88
|
+
elsif plugin_ids = qs['id'] and plugin_id = plugin_ids[0]
|
89
|
+
# ?id= to search a plugin by 'id <plugin_id>' config param
|
90
|
+
if obj = @agent.plugin_info_by_id(plugin_id, opts)
|
91
|
+
list = [obj]
|
92
|
+
else
|
93
|
+
list = []
|
94
|
+
end
|
108
95
|
|
109
|
-
|
110
|
-
|
111
|
-
|
96
|
+
elsif types = qs['type'] and type = types[0]
|
97
|
+
# ?type= to search plugins by 'type <type>' config param
|
98
|
+
list = @agent.plugins_info_by_type(type, opts)
|
112
99
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
100
|
+
else
|
101
|
+
# otherwise show all plugins
|
102
|
+
list = @agent.plugins_info_all(opts)
|
103
|
+
end
|
104
|
+
|
105
|
+
return list, opts
|
106
|
+
end
|
107
|
+
|
108
|
+
def render_json(obj, opts={})
|
109
|
+
render_json_error(200, obj, opts)
|
110
|
+
end
|
111
|
+
|
112
|
+
def render_json_error(code, obj, opts={})
|
113
|
+
if opts[:pretty_json]
|
114
|
+
js = JSON.pretty_generate(obj)
|
115
|
+
else
|
116
|
+
js = obj.to_json
|
117
|
+
end
|
118
|
+
[code, {'Content-Type'=>'application/json'}, js]
|
118
119
|
end
|
119
|
-
[code, {'Content-Type'=>'application/json'}, js]
|
120
120
|
end
|
121
|
-
end
|
122
121
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
122
|
+
class LTSVMonitorServlet < MonitorServlet
|
123
|
+
def process(req, res)
|
124
|
+
list, opts = build_object(req, res)
|
125
|
+
return unless list
|
127
126
|
|
128
|
-
|
127
|
+
normalized = JSON.parse(list.to_json)
|
129
128
|
|
130
|
-
|
129
|
+
text = ''
|
131
130
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
131
|
+
normalized.map {|hash|
|
132
|
+
row = []
|
133
|
+
hash.each_pair {|k,v|
|
134
|
+
unless v.is_a?(Hash) || v.is_a?(Array)
|
135
|
+
row << "#{k}:#{v}"
|
136
|
+
end
|
137
|
+
}
|
138
|
+
text << row.join("\t") << "\n"
|
138
139
|
}
|
139
|
-
text << row.join("\t") << "\n"
|
140
|
-
}
|
141
140
|
|
142
|
-
|
141
|
+
[200, {'Content-Type'=>'text/plain'}, text]
|
142
|
+
end
|
143
143
|
end
|
144
|
-
end
|
145
144
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
145
|
+
class JSONMonitorServlet < MonitorServlet
|
146
|
+
def process(req, res)
|
147
|
+
list, opts = build_object(req, res)
|
148
|
+
return unless list
|
150
149
|
|
151
|
-
|
152
|
-
|
153
|
-
|
150
|
+
render_json({
|
151
|
+
'plugins' => list
|
152
|
+
}, opts)
|
153
|
+
end
|
154
154
|
end
|
155
|
-
end
|
156
155
|
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
end
|
171
|
-
|
172
|
-
def shutdown
|
173
|
-
if @srv
|
174
|
-
@srv.shutdown
|
175
|
-
@srv = nil
|
156
|
+
def start
|
157
|
+
$log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
|
158
|
+
@srv = WEBrick::HTTPServer.new({
|
159
|
+
:BindAddress => @bind,
|
160
|
+
:Port => @port,
|
161
|
+
:Logger => WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
|
162
|
+
:AccessLog => [],
|
163
|
+
})
|
164
|
+
@srv.mount('/api/plugins', LTSVMonitorServlet, self)
|
165
|
+
@srv.mount('/api/plugins.json', JSONMonitorServlet, self)
|
166
|
+
@thread = Thread.new {
|
167
|
+
@srv.start
|
168
|
+
}
|
176
169
|
end
|
177
|
-
|
178
|
-
|
179
|
-
@
|
170
|
+
|
171
|
+
def shutdown
|
172
|
+
if @srv
|
173
|
+
@srv.shutdown
|
174
|
+
@srv = nil
|
175
|
+
end
|
176
|
+
if @thread
|
177
|
+
@thread.join
|
178
|
+
@thread = nil
|
179
|
+
end
|
180
180
|
end
|
181
|
-
end
|
182
181
|
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
}
|
192
|
-
|
193
|
-
def all_plugins
|
194
|
-
array = []
|
195
|
-
|
196
|
-
# get all input plugins
|
197
|
-
array.concat Engine.sources
|
198
|
-
|
199
|
-
# get all output plugins
|
200
|
-
Engine.matches.each {|m|
|
201
|
-
MonitorAgentInput.collect_children(m.output, array)
|
182
|
+
MONITOR_INFO = {
|
183
|
+
'plugin_id' => 'plugin_id',
|
184
|
+
'type' => 'config["type"]',
|
185
|
+
'output_plugin' => 'is_a?(::Fluent::Output)',
|
186
|
+
'buffer_queue_length' => '@buffer.queue_size',
|
187
|
+
'buffer_total_queued_size' => '@buffer.total_queued_chunk_size',
|
188
|
+
'retry_count' => '@error_history.size',
|
189
|
+
'config' => 'config',
|
202
190
|
}
|
203
191
|
|
204
|
-
|
205
|
-
|
192
|
+
def all_plugins
|
193
|
+
array = []
|
194
|
+
|
195
|
+
# get all input plugins
|
196
|
+
array.concat Engine.sources
|
206
197
|
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
array << pe
|
211
|
-
if pe.is_a?(MultiOutput) && pe.respond_to?(:outputs)
|
212
|
-
pe.outputs.each {|nop|
|
213
|
-
collect_children(nop, array)
|
198
|
+
# get all output plugins
|
199
|
+
Engine.matches.each {|m|
|
200
|
+
MonitorAgentInput.collect_children(m.output, array)
|
214
201
|
}
|
202
|
+
|
203
|
+
array
|
215
204
|
end
|
216
|
-
array
|
217
|
-
end
|
218
205
|
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
206
|
+
# get nexted plugins (such as <store> of the copy plugin)
|
207
|
+
# from the plugin `pe` recursively
|
208
|
+
def self.collect_children(pe, array=[])
|
209
|
+
array << pe
|
210
|
+
if pe.is_a?(MultiOutput) && pe.respond_to?(:outputs)
|
211
|
+
pe.outputs.each {|nop|
|
212
|
+
collect_children(nop, array)
|
213
|
+
}
|
214
|
+
end
|
215
|
+
array
|
228
216
|
end
|
229
|
-
end
|
230
217
|
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
218
|
+
# try to match the tag and get the info from the
|
219
|
+
# matched output plugin
|
220
|
+
def plugin_info_by_tag(tag, opts={})
|
221
|
+
m = Engine.match(tag)
|
222
|
+
if m
|
223
|
+
pe = m.output
|
224
|
+
get_monitor_info(pe, opts)
|
225
|
+
else
|
226
|
+
nil
|
227
|
+
end
|
240
228
|
end
|
241
|
-
end
|
242
229
|
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
230
|
+
# search a plugin by plugin_id
|
231
|
+
def plugin_info_by_id(plugin_id, opts={})
|
232
|
+
found = all_plugins.find {|pe|
|
233
|
+
pe.respond_to?(:plugin_id) && pe.plugin_id.to_s == plugin_id
|
234
|
+
}
|
235
|
+
if found
|
236
|
+
get_monitor_info(found, opts)
|
237
|
+
else
|
238
|
+
nil
|
239
|
+
end
|
240
|
+
end
|
253
241
|
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
242
|
+
# This method returns an array because
|
243
|
+
# multiple plugins could have the same type
|
244
|
+
def plugins_info_by_type(type, opts={})
|
245
|
+
array = all_plugins.select {|pe|
|
246
|
+
pe.config['type'] == type rescue nil
|
247
|
+
}
|
248
|
+
array.map {|pe|
|
249
|
+
get_monitor_info(pe, opts)
|
250
|
+
}
|
251
|
+
end
|
259
252
|
|
260
|
-
|
261
|
-
|
262
|
-
|
253
|
+
def plugins_info_all(opts={})
|
254
|
+
all_plugins.map {|pe|
|
255
|
+
get_monitor_info(pe, opts)
|
256
|
+
}
|
257
|
+
end
|
263
258
|
|
264
|
-
#
|
265
|
-
|
266
|
-
|
267
|
-
obj[key] = pe.instance_eval(code)
|
268
|
-
rescue
|
269
|
-
end
|
270
|
-
}
|
259
|
+
# get monitor info from the plugin `pe` and return a hash object
|
260
|
+
def get_monitor_info(pe, opts={})
|
261
|
+
obj = {}
|
271
262
|
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
263
|
+
# run MONITOR_INFO in plugins' instance context and store the info to obj
|
264
|
+
MONITOR_INFO.each_pair {|key,code|
|
265
|
+
begin
|
266
|
+
obj[key] = pe.instance_eval(code)
|
267
|
+
rescue
|
268
|
+
end
|
269
|
+
}
|
270
|
+
|
271
|
+
# include all instance variables if :with_debug_info is set
|
272
|
+
if opts[:with_debug_info]
|
273
|
+
iv = {}
|
274
|
+
pe.instance_eval do
|
275
|
+
instance_variables.each {|sym|
|
276
|
+
key = sym.to_s[1..-1] # removes first '@'
|
277
|
+
iv[key] = instance_variable_get(sym)
|
278
|
+
}
|
279
|
+
end
|
280
|
+
obj['instance_variables'] = iv
|
280
281
|
end
|
281
|
-
obj['instance_variables'] = iv
|
282
|
-
end
|
283
282
|
|
284
|
-
|
283
|
+
obj
|
284
|
+
end
|
285
285
|
end
|
286
286
|
end
|
287
|
-
|
288
|
-
end
|