fluentd 0.10.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- data/AUTHORS +1 -0
- data/COPYING +14 -0
- data/ChangeLog +178 -0
- data/README.rdoc +57 -0
- data/Rakefile +62 -0
- data/VERSION +1 -0
- data/bin/fluent-cat +6 -0
- data/bin/fluent-gem +10 -0
- data/bin/fluentd +6 -0
- data/fluent.conf +78 -0
- data/fluentd.gemspec +116 -0
- data/lib/fluent/buffer.rb +274 -0
- data/lib/fluent/command/cat.rb +299 -0
- data/lib/fluent/command/fluentd.rb +245 -0
- data/lib/fluent/config.rb +304 -0
- data/lib/fluent/engine.rb +224 -0
- data/lib/fluent/env.rb +6 -0
- data/lib/fluent/event.rb +159 -0
- data/lib/fluent/input.rb +41 -0
- data/lib/fluent/load.rb +23 -0
- data/lib/fluent/log.rb +277 -0
- data/lib/fluent/match.rb +189 -0
- data/lib/fluent/mixin.rb +170 -0
- data/lib/fluent/output.rb +466 -0
- data/lib/fluent/parser.rb +115 -0
- data/lib/fluent/plugin.rb +145 -0
- data/lib/fluent/plugin/buf_file.rb +181 -0
- data/lib/fluent/plugin/buf_memory.rb +97 -0
- data/lib/fluent/plugin/buf_zfile.rb +84 -0
- data/lib/fluent/plugin/in_http.rb +282 -0
- data/lib/fluent/plugin/in_stream.rb +187 -0
- data/lib/fluent/plugin/in_syslog.rb +174 -0
- data/lib/fluent/plugin/in_tail.rb +150 -0
- data/lib/fluent/plugin/out_copy.rb +72 -0
- data/lib/fluent/plugin/out_file.rb +111 -0
- data/lib/fluent/plugin/out_null.rb +44 -0
- data/lib/fluent/plugin/out_roundrobin.rb +72 -0
- data/lib/fluent/plugin/out_stdout.rb +34 -0
- data/lib/fluent/plugin/out_stream.rb +128 -0
- data/lib/fluent/plugin/out_test.rb +68 -0
- data/lib/fluent/test.rb +8 -0
- data/lib/fluent/test/base.rb +63 -0
- data/lib/fluent/test/input_test.rb +89 -0
- data/lib/fluent/test/output_test.rb +93 -0
- data/lib/fluent/version.rb +5 -0
- data/test/helper.rb +6 -0
- data/test/match.rb +115 -0
- data/test/plugin/in_http.rb +84 -0
- data/test/plugin/in_stream.rb +136 -0
- data/test/plugin/out_copy.rb +55 -0
- data/test/plugin/out_file.rb +82 -0
- data/test/plugin/out_roundrobin.rb +65 -0
- data/test/plugin/out_stream.rb +74 -0
- metadata +224 -0
data/lib/fluent/log.rb
ADDED
@@ -0,0 +1,277 @@
|
|
1
|
+
#
|
2
|
+
# Fluent
|
3
|
+
#
|
4
|
+
# Copyright (C) 2011 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
module Fluent
|
19
|
+
|
20
|
+
|
21
|
+
class Log
|
22
|
+
module TTYColor
|
23
|
+
RESET = "\033]R"
|
24
|
+
CRE = "\033[K"
|
25
|
+
CLEAR = "\033c"
|
26
|
+
NORMAL = "\033[0;39m"
|
27
|
+
RED = "\033[1;31m"
|
28
|
+
GREEN = "\033[1;32m"
|
29
|
+
YELLOW = "\033[1;33m"
|
30
|
+
BLUE = "\033[1;34m"
|
31
|
+
MAGENTA = "\033[1;35m"
|
32
|
+
CYAN = "\033[1;36m"
|
33
|
+
WHITE = "\033[1;37m"
|
34
|
+
end
|
35
|
+
|
36
|
+
LEVEL_TRACE = 0
|
37
|
+
LEVEL_DEBUG = 1
|
38
|
+
LEVEL_INFO = 2
|
39
|
+
LEVEL_WARN = 3
|
40
|
+
LEVEL_ERROR = 4
|
41
|
+
LEVEL_FATAL = 5
|
42
|
+
|
43
|
+
def initialize(out=STDERR, level=LEVEL_TRACE)
|
44
|
+
@out = out
|
45
|
+
@level = level
|
46
|
+
@debug_mode = false
|
47
|
+
@self_event = false
|
48
|
+
@tag = 'fluent'
|
49
|
+
@time_format = '%Y-%m-%d %H:%M:%S %z: '
|
50
|
+
enable_color out.tty?
|
51
|
+
end
|
52
|
+
|
53
|
+
attr_accessor :out
|
54
|
+
attr_accessor :level
|
55
|
+
attr_accessor :tag
|
56
|
+
attr_accessor :time_format
|
57
|
+
|
58
|
+
def enable_debug(b=true)
|
59
|
+
@debug_mode = b
|
60
|
+
self
|
61
|
+
end
|
62
|
+
|
63
|
+
def enable_event(b=true)
|
64
|
+
@self_event = b
|
65
|
+
self
|
66
|
+
end
|
67
|
+
|
68
|
+
def enable_color(b=true)
|
69
|
+
if b
|
70
|
+
@color_trace = TTYColor::BLUE
|
71
|
+
@color_debug = TTYColor::WHITE
|
72
|
+
@color_info = TTYColor::GREEN
|
73
|
+
@color_warn = TTYColor::YELLOW
|
74
|
+
@color_error = TTYColor::MAGENTA
|
75
|
+
@color_fatal = TTYColor::RED
|
76
|
+
@color_reset = TTYColor::NORMAL
|
77
|
+
else
|
78
|
+
@color_trace = ''
|
79
|
+
@color_debug = ''
|
80
|
+
@color_info = ''
|
81
|
+
@color_warn = ''
|
82
|
+
@color_error = ''
|
83
|
+
@color_fatal = ''
|
84
|
+
@color_reset = ''
|
85
|
+
end
|
86
|
+
self
|
87
|
+
end
|
88
|
+
|
89
|
+
def on_trace(&block)
|
90
|
+
return if @level > LEVEL_TRACE
|
91
|
+
block.call if block
|
92
|
+
end
|
93
|
+
|
94
|
+
def trace(*args, &block)
|
95
|
+
return if @level > LEVEL_TRACE
|
96
|
+
args << block.call if block
|
97
|
+
time, msg = event(:trace, args)
|
98
|
+
puts [@color_trace, caller_line(time, 1), msg, @color_reset].join
|
99
|
+
end
|
100
|
+
alias TRACE trace
|
101
|
+
|
102
|
+
def trace_backtrace(backtrace=$!.backtrace)
|
103
|
+
return if @level > LEVEL_TRACE
|
104
|
+
time = Time.now
|
105
|
+
backtrace.each {|msg|
|
106
|
+
puts [" ", caller_line(time,4), msg].join
|
107
|
+
}
|
108
|
+
nil
|
109
|
+
end
|
110
|
+
|
111
|
+
def on_debug(&block)
|
112
|
+
return if @level > LEVEL_DEBUG
|
113
|
+
block.call if block
|
114
|
+
end
|
115
|
+
|
116
|
+
def debug(*args, &block)
|
117
|
+
return if @level > LEVEL_DEBUG
|
118
|
+
args << block.call if block
|
119
|
+
time, msg = event(:debug, args)
|
120
|
+
puts [@color_debug, caller_line(time, 1), msg, @color_reset].join
|
121
|
+
end
|
122
|
+
alias DEBUG debug
|
123
|
+
|
124
|
+
def debug_backtrace(backtrace=$!.backtrace)
|
125
|
+
return if @level > LEVEL_DEBUG
|
126
|
+
time = Time.now
|
127
|
+
backtrace.each {|msg|
|
128
|
+
puts [" ", caller_line(time,4), msg].join
|
129
|
+
}
|
130
|
+
nil
|
131
|
+
end
|
132
|
+
|
133
|
+
def on_info(&block)
|
134
|
+
return if @level > LEVEL_INFO
|
135
|
+
block.call if block
|
136
|
+
end
|
137
|
+
|
138
|
+
def info(*args, &block)
|
139
|
+
return if @level > LEVEL_INFO
|
140
|
+
args << block.call if block
|
141
|
+
time, msg = event(:info, args)
|
142
|
+
puts [@color_info, caller_line(time, 1), msg, @color_reset].join
|
143
|
+
end
|
144
|
+
alias INFO info
|
145
|
+
|
146
|
+
def info_backtrace(backtrace=$!.backtrace)
|
147
|
+
return if @level > LEVEL_INFO
|
148
|
+
time = Time.now
|
149
|
+
backtrace.each {|msg|
|
150
|
+
puts [" ", caller_line(time,4), msg].join
|
151
|
+
}
|
152
|
+
nil
|
153
|
+
end
|
154
|
+
|
155
|
+
def on_warn(&block)
|
156
|
+
return if @level > LEVEL_WARN
|
157
|
+
block.call if block
|
158
|
+
end
|
159
|
+
|
160
|
+
def warn(*args, &block)
|
161
|
+
return if @level > LEVEL_WARN
|
162
|
+
args << block.call if block
|
163
|
+
time, msg = event(:warn, args)
|
164
|
+
puts [@color_warn, caller_line(time, 1), msg, @color_reset].join
|
165
|
+
end
|
166
|
+
alias WARN warn
|
167
|
+
|
168
|
+
def warn_backtrace(backtrace=$!.backtrace)
|
169
|
+
return if @level > LEVEL_WARN
|
170
|
+
time = Time.now
|
171
|
+
backtrace.each {|msg|
|
172
|
+
puts [" ", caller_line(time,4), msg].join
|
173
|
+
}
|
174
|
+
nil
|
175
|
+
end
|
176
|
+
|
177
|
+
def on_error(&block)
|
178
|
+
return if @level > LEVEL_ERROR
|
179
|
+
block.call if block
|
180
|
+
end
|
181
|
+
|
182
|
+
def error(*args, &block)
|
183
|
+
return if @level > LEVEL_ERROR
|
184
|
+
args << block.call if block
|
185
|
+
time, msg = event(:error, args)
|
186
|
+
puts [@color_error, caller_line(time, 1), msg, @color_reset].join
|
187
|
+
end
|
188
|
+
alias ERROR error
|
189
|
+
|
190
|
+
def error_backtrace(backtrace=$!.backtrace)
|
191
|
+
return if @level > LEVEL_ERROR
|
192
|
+
time = Time.now
|
193
|
+
backtrace.each {|msg|
|
194
|
+
puts [" ", caller_line(time,4), msg].join
|
195
|
+
}
|
196
|
+
nil
|
197
|
+
end
|
198
|
+
|
199
|
+
def on_fatal(&block)
|
200
|
+
return if @level > LEVEL_FATAL
|
201
|
+
block.call if block
|
202
|
+
end
|
203
|
+
|
204
|
+
def fatal(*args, &block)
|
205
|
+
return if @level > LEVEL_FATAL
|
206
|
+
args << block.call if block
|
207
|
+
time, msg = event(:fatal, args)
|
208
|
+
puts [@color_fatal, caller_line(time, 1), msg, @color_reset].join
|
209
|
+
end
|
210
|
+
alias FATAL fatal
|
211
|
+
|
212
|
+
def fatal_backtrace(backtrace=$!.backtrace)
|
213
|
+
return if @level > LEVEL_FATAL
|
214
|
+
time = Time.now
|
215
|
+
backtrace.each {|msg|
|
216
|
+
puts [" ", caller_line(time,4), msg].join
|
217
|
+
}
|
218
|
+
nil
|
219
|
+
end
|
220
|
+
|
221
|
+
def puts(msg)
|
222
|
+
@out.puts(msg)
|
223
|
+
@out.flush
|
224
|
+
msg
|
225
|
+
rescue
|
226
|
+
# FIXME
|
227
|
+
nil
|
228
|
+
end
|
229
|
+
|
230
|
+
private
|
231
|
+
def event(level, args)
|
232
|
+
time = Time.now
|
233
|
+
message = ''
|
234
|
+
record = {'message'=>message}
|
235
|
+
args.each {|a|
|
236
|
+
if a.is_a?(Hash)
|
237
|
+
a.each_pair {|k,v|
|
238
|
+
record[k.to_s] = v
|
239
|
+
}
|
240
|
+
else
|
241
|
+
message << a.to_s
|
242
|
+
end
|
243
|
+
}
|
244
|
+
|
245
|
+
if @self_event
|
246
|
+
c = caller
|
247
|
+
if c.count(c.shift) <= 0
|
248
|
+
Engine.emit("#{@tag}.#{level}", time.to_i, record)
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
record.each_pair {|k,v|
|
253
|
+
if v.object_id != message.object_id
|
254
|
+
message << " #{k}=#{v.inspect}"
|
255
|
+
end
|
256
|
+
}
|
257
|
+
|
258
|
+
return time, message
|
259
|
+
end
|
260
|
+
|
261
|
+
def caller_line(time, level)
|
262
|
+
line = caller(level+1)[0]
|
263
|
+
if @debug_mode
|
264
|
+
if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
|
265
|
+
file = match[1].split('/')[-2,2].join('/')
|
266
|
+
line = match[2]
|
267
|
+
method = match[3]
|
268
|
+
return "#{time.strftime(@time_format)}#{file}:#{line}:#{method}: "
|
269
|
+
end
|
270
|
+
end
|
271
|
+
return time.strftime(@time_format)
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
|
276
|
+
end
|
277
|
+
|
data/lib/fluent/match.rb
ADDED
@@ -0,0 +1,189 @@
|
|
1
|
+
#
|
2
|
+
# Fluent
|
3
|
+
#
|
4
|
+
# Copyright (C) 2011 FURUHASHI Sadayuki
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
module Fluent
|
19
|
+
|
20
|
+
|
21
|
+
class Match
|
22
|
+
def initialize(pattern_str, output)
|
23
|
+
patterns = pattern_str.split(/\s+/).map {|str|
|
24
|
+
MatchPattern.create(str)
|
25
|
+
}
|
26
|
+
if patterns.length == 1
|
27
|
+
@pattern = patterns[0]
|
28
|
+
else
|
29
|
+
@pattern = OrMatchPattern.new(patterns)
|
30
|
+
end
|
31
|
+
@output = output
|
32
|
+
end
|
33
|
+
|
34
|
+
attr_reader :output
|
35
|
+
|
36
|
+
def emit(tag, es)
|
37
|
+
chain = NullOutputChain.instance
|
38
|
+
@output.emit(tag, es, chain)
|
39
|
+
end
|
40
|
+
|
41
|
+
def start
|
42
|
+
@output.start
|
43
|
+
end
|
44
|
+
|
45
|
+
def shutdown
|
46
|
+
@output.shutdown
|
47
|
+
end
|
48
|
+
|
49
|
+
def match(tag)
|
50
|
+
if @pattern.match(tag)
|
51
|
+
return true
|
52
|
+
end
|
53
|
+
return false
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
class MatchPattern
|
59
|
+
def self.create(str)
|
60
|
+
GlobMatchPattern.new(str)
|
61
|
+
end
|
62
|
+
|
63
|
+
#def match(str)
|
64
|
+
#end
|
65
|
+
end
|
66
|
+
|
67
|
+
## TODO
|
68
|
+
#class RegexMatchPattern < MatchPattern
|
69
|
+
# def initialize(regex)
|
70
|
+
# @regex = regex
|
71
|
+
# end
|
72
|
+
#
|
73
|
+
# def match(str)
|
74
|
+
# @regex.match(str) != nil
|
75
|
+
# end
|
76
|
+
#end
|
77
|
+
|
78
|
+
class GlobMatchPattern < MatchPattern
|
79
|
+
def initialize(pat)
|
80
|
+
stack = []
|
81
|
+
regex = ['']
|
82
|
+
escape = false
|
83
|
+
dot = false
|
84
|
+
|
85
|
+
i = 0
|
86
|
+
while i < pat.length
|
87
|
+
c = pat[i,1]
|
88
|
+
|
89
|
+
if escape
|
90
|
+
regex.last << Regexp.escape(c)
|
91
|
+
escape = false
|
92
|
+
i += 1
|
93
|
+
next
|
94
|
+
|
95
|
+
elsif pat[i,2] == "**"
|
96
|
+
# recursive any
|
97
|
+
if dot
|
98
|
+
regex.last << "(?![^\\.])"
|
99
|
+
dot = false
|
100
|
+
end
|
101
|
+
if pat[i+2,1] == "."
|
102
|
+
regex.last << "(?:.*\\.|\\A)"
|
103
|
+
i += 3
|
104
|
+
else
|
105
|
+
regex.last << ".*"
|
106
|
+
i += 2
|
107
|
+
end
|
108
|
+
next
|
109
|
+
|
110
|
+
elsif dot
|
111
|
+
regex.last << "\\."
|
112
|
+
dot = false
|
113
|
+
end
|
114
|
+
|
115
|
+
if c == "\\"
|
116
|
+
escape = true
|
117
|
+
|
118
|
+
elsif c == "."
|
119
|
+
dot = true
|
120
|
+
|
121
|
+
elsif c == "*"
|
122
|
+
# any
|
123
|
+
regex.last << "[^\\.]*"
|
124
|
+
|
125
|
+
# TODO
|
126
|
+
#elsif c == "["
|
127
|
+
# # character class
|
128
|
+
# chars = ''
|
129
|
+
# while i < pat.length
|
130
|
+
# c = pat[i,1]
|
131
|
+
# if c == "]"
|
132
|
+
# break
|
133
|
+
# else
|
134
|
+
# chars << c
|
135
|
+
# end
|
136
|
+
# i += 1
|
137
|
+
# end
|
138
|
+
# regex.last << '['+Regexp.escape(chars).gsub("\\-",'-')+']'
|
139
|
+
|
140
|
+
elsif c == "{"
|
141
|
+
# or
|
142
|
+
stack.push []
|
143
|
+
regex.push ''
|
144
|
+
|
145
|
+
elsif c == "}" && !stack.empty?
|
146
|
+
stack.last << regex.pop
|
147
|
+
regex.last << Regexp.union(*stack.pop.map {|r| Regexp.new(r) }).to_s
|
148
|
+
|
149
|
+
elsif c == "," && !stack.empty?
|
150
|
+
stack.last << regex.pop
|
151
|
+
regex.push ''
|
152
|
+
|
153
|
+
elsif c =~ /[a-zA-Z0-9_]/
|
154
|
+
regex.last << c
|
155
|
+
|
156
|
+
else
|
157
|
+
regex.last << "\\#{c}"
|
158
|
+
end
|
159
|
+
|
160
|
+
i += 1
|
161
|
+
end
|
162
|
+
|
163
|
+
until stack.empty?
|
164
|
+
stack.last << regex.pop
|
165
|
+
regex.last << Regexp.union(*stack.pop).to_s
|
166
|
+
end
|
167
|
+
|
168
|
+
@regex = Regexp.new("\\A"+regex.last+"\\Z")
|
169
|
+
end
|
170
|
+
|
171
|
+
def match(str)
|
172
|
+
@regex.match(str) != nil
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
|
177
|
+
class OrMatchPattern < MatchPattern
|
178
|
+
def initialize(patterns)
|
179
|
+
@patterns = patterns
|
180
|
+
end
|
181
|
+
|
182
|
+
def match(str)
|
183
|
+
@patterns.any? {|pattern| pattern.match(str) }
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
|
188
|
+
end
|
189
|
+
|