fluentd 0.10.36 → 0.10.37
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +7 -0
- data/.travis.yml +2 -2
- data/ChangeLog +8 -0
- data/lib/fluent/config.rb +285 -289
- data/lib/fluent/config_dsl.rb +74 -0
- data/lib/fluent/engine.rb +250 -247
- data/lib/fluent/env.rb +5 -5
- data/lib/fluent/event.rb +120 -124
- data/lib/fluent/input.rb +13 -17
- data/lib/fluent/load.rb +1 -0
- data/lib/fluent/log.rb +238 -242
- data/lib/fluent/match.rb +132 -137
- data/lib/fluent/mixin.rb +151 -155
- data/lib/fluent/parser.rb +201 -205
- data/lib/fluent/plugin.rb +117 -121
- data/lib/fluent/plugin/buf_file.rb +8 -0
- data/lib/fluent/plugin/exec_util.rb +49 -0
- data/lib/fluent/plugin/in_exec.rb +44 -30
- data/lib/fluent/plugin/in_forward.rb +4 -2
- data/lib/fluent/plugin/in_http.rb +5 -0
- data/lib/fluent/plugin/in_stream.rb +5 -2
- data/lib/fluent/plugin/out_exec_filter.rb +4 -47
- data/lib/fluent/plugin/out_stdout.rb +21 -1
- data/lib/fluent/process.rb +358 -362
- data/lib/fluent/status.rb +25 -30
- data/lib/fluent/supervisor.rb +277 -281
- data/lib/fluent/test/base.rb +35 -39
- data/lib/fluent/test/input_test.rb +68 -63
- data/lib/fluent/test/output_test.rb +98 -101
- data/lib/fluent/version.rb +1 -1
- data/test/configdsl.rb +77 -0
- data/test/plugin/in_exec.rb +73 -13
- data/test/plugin/in_gc_stat.rb +1 -1
- data/test/plugin/in_object_space.rb +2 -2
- data/test/plugin/out_stdout.rb +45 -2
- data/test/scripts/exec_script.rb +26 -0
- metadata +31 -46
data/lib/fluent/match.rb
CHANGED
@@ -16,174 +16,169 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
+
class Match
|
20
|
+
def initialize(pattern_str, output)
|
21
|
+
patterns = pattern_str.split(/\s+/).map {|str|
|
22
|
+
MatchPattern.create(str)
|
23
|
+
}
|
24
|
+
if patterns.length == 1
|
25
|
+
@pattern = patterns[0]
|
26
|
+
else
|
27
|
+
@pattern = OrMatchPattern.new(patterns)
|
28
|
+
end
|
29
|
+
@output = output
|
30
|
+
end
|
19
31
|
|
32
|
+
attr_reader :output
|
20
33
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
MatchPattern.create(str)
|
25
|
-
}
|
26
|
-
if patterns.length == 1
|
27
|
-
@pattern = patterns[0]
|
28
|
-
else
|
29
|
-
@pattern = OrMatchPattern.new(patterns)
|
34
|
+
def emit(tag, es)
|
35
|
+
chain = NullOutputChain.instance
|
36
|
+
@output.emit(tag, es, chain)
|
30
37
|
end
|
31
|
-
@output = output
|
32
|
-
end
|
33
38
|
|
34
|
-
|
39
|
+
def start
|
40
|
+
@output.start
|
41
|
+
end
|
35
42
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
end
|
43
|
+
def shutdown
|
44
|
+
@output.shutdown
|
45
|
+
end
|
40
46
|
|
41
|
-
|
42
|
-
|
47
|
+
def match(tag)
|
48
|
+
if @pattern.match(tag)
|
49
|
+
return true
|
50
|
+
end
|
51
|
+
return false
|
52
|
+
end
|
43
53
|
end
|
44
54
|
|
45
|
-
def shutdown
|
46
|
-
@output.shutdown
|
47
|
-
end
|
48
55
|
|
49
|
-
|
50
|
-
|
51
|
-
|
56
|
+
class MatchPattern
|
57
|
+
def self.create(str)
|
58
|
+
GlobMatchPattern.new(str)
|
52
59
|
end
|
53
|
-
return false
|
54
|
-
end
|
55
|
-
end
|
56
60
|
|
57
|
-
|
58
|
-
|
59
|
-
def self.create(str)
|
60
|
-
GlobMatchPattern.new(str)
|
61
|
+
#def match(str)
|
62
|
+
#end
|
61
63
|
end
|
62
64
|
|
63
|
-
|
65
|
+
## TODO
|
66
|
+
#class RegexMatchPattern < MatchPattern
|
67
|
+
# def initialize(regex)
|
68
|
+
# @regex = regex
|
69
|
+
# end
|
70
|
+
#
|
71
|
+
# def match(str)
|
72
|
+
# @regex.match(str) != nil
|
73
|
+
# end
|
64
74
|
#end
|
65
|
-
end
|
66
75
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
76
|
+
class GlobMatchPattern < MatchPattern
|
77
|
+
def initialize(pat)
|
78
|
+
stack = []
|
79
|
+
regex = ['']
|
80
|
+
escape = false
|
81
|
+
dot = false
|
82
|
+
|
83
|
+
i = 0
|
84
|
+
while i < pat.length
|
85
|
+
c = pat[i,1]
|
86
|
+
|
87
|
+
if escape
|
88
|
+
regex.last << Regexp.escape(c)
|
89
|
+
escape = false
|
90
|
+
i += 1
|
91
|
+
next
|
92
|
+
|
93
|
+
elsif pat[i,2] == "**"
|
94
|
+
# recursive any
|
95
|
+
if dot
|
96
|
+
regex.last << "(?![^\\.])"
|
97
|
+
dot = false
|
98
|
+
end
|
99
|
+
if pat[i+2,1] == "."
|
100
|
+
regex.last << "(?:.*\\.|\\A)"
|
101
|
+
i += 3
|
102
|
+
else
|
103
|
+
regex.last << ".*"
|
104
|
+
i += 2
|
105
|
+
end
|
106
|
+
next
|
107
|
+
|
108
|
+
elsif dot
|
109
|
+
regex.last << "\\."
|
99
110
|
dot = false
|
100
111
|
end
|
101
|
-
|
102
|
-
|
103
|
-
|
112
|
+
|
113
|
+
if c == "\\"
|
114
|
+
escape = true
|
115
|
+
|
116
|
+
elsif c == "."
|
117
|
+
dot = true
|
118
|
+
|
119
|
+
elsif c == "*"
|
120
|
+
# any
|
121
|
+
regex.last << "[^\\.]*"
|
122
|
+
|
123
|
+
# TODO
|
124
|
+
#elsif c == "["
|
125
|
+
# # character class
|
126
|
+
# chars = ''
|
127
|
+
# while i < pat.length
|
128
|
+
# c = pat[i,1]
|
129
|
+
# if c == "]"
|
130
|
+
# break
|
131
|
+
# else
|
132
|
+
# chars << c
|
133
|
+
# end
|
134
|
+
# i += 1
|
135
|
+
# end
|
136
|
+
# regex.last << '['+Regexp.escape(chars).gsub("\\-",'-')+']'
|
137
|
+
|
138
|
+
elsif c == "{"
|
139
|
+
# or
|
140
|
+
stack.push []
|
141
|
+
regex.push ''
|
142
|
+
|
143
|
+
elsif c == "}" && !stack.empty?
|
144
|
+
stack.last << regex.pop
|
145
|
+
regex.last << Regexp.union(*stack.pop.map {|r| Regexp.new(r) }).to_s
|
146
|
+
|
147
|
+
elsif c == "," && !stack.empty?
|
148
|
+
stack.last << regex.pop
|
149
|
+
regex.push ''
|
150
|
+
|
151
|
+
elsif c =~ /[a-zA-Z0-9_]/
|
152
|
+
regex.last << c
|
153
|
+
|
104
154
|
else
|
105
|
-
regex.last << "
|
106
|
-
i += 2
|
155
|
+
regex.last << "\\#{c}"
|
107
156
|
end
|
108
|
-
next
|
109
157
|
|
110
|
-
|
111
|
-
regex.last << "\\."
|
112
|
-
dot = false
|
158
|
+
i += 1
|
113
159
|
end
|
114
160
|
|
115
|
-
|
116
|
-
escape = true
|
117
|
-
|
118
|
-
elsif c == "."
|
119
|
-
dot = true
|
120
|
-
|
121
|
-
elsif c == "*"
|
122
|
-
# any
|
123
|
-
regex.last << "[^\\.]*"
|
124
|
-
|
125
|
-
# TODO
|
126
|
-
#elsif c == "["
|
127
|
-
# # character class
|
128
|
-
# chars = ''
|
129
|
-
# while i < pat.length
|
130
|
-
# c = pat[i,1]
|
131
|
-
# if c == "]"
|
132
|
-
# break
|
133
|
-
# else
|
134
|
-
# chars << c
|
135
|
-
# end
|
136
|
-
# i += 1
|
137
|
-
# end
|
138
|
-
# regex.last << '['+Regexp.escape(chars).gsub("\\-",'-')+']'
|
139
|
-
|
140
|
-
elsif c == "{"
|
141
|
-
# or
|
142
|
-
stack.push []
|
143
|
-
regex.push ''
|
144
|
-
|
145
|
-
elsif c == "}" && !stack.empty?
|
161
|
+
until stack.empty?
|
146
162
|
stack.last << regex.pop
|
147
|
-
regex.last << Regexp.union(*stack.pop
|
148
|
-
|
149
|
-
elsif c == "," && !stack.empty?
|
150
|
-
stack.last << regex.pop
|
151
|
-
regex.push ''
|
152
|
-
|
153
|
-
elsif c =~ /[a-zA-Z0-9_]/
|
154
|
-
regex.last << c
|
155
|
-
|
156
|
-
else
|
157
|
-
regex.last << "\\#{c}"
|
163
|
+
regex.last << Regexp.union(*stack.pop).to_s
|
158
164
|
end
|
159
165
|
|
160
|
-
|
166
|
+
@regex = Regexp.new("\\A"+regex.last+"\\Z")
|
161
167
|
end
|
162
168
|
|
163
|
-
|
164
|
-
|
165
|
-
regex.last << Regexp.union(*stack.pop).to_s
|
169
|
+
def match(str)
|
170
|
+
@regex.match(str) != nil
|
166
171
|
end
|
167
|
-
|
168
|
-
@regex = Regexp.new("\\A"+regex.last+"\\Z")
|
169
|
-
end
|
170
|
-
|
171
|
-
def match(str)
|
172
|
-
@regex.match(str) != nil
|
173
172
|
end
|
174
|
-
end
|
175
173
|
|
176
174
|
|
177
|
-
class OrMatchPattern < MatchPattern
|
178
|
-
|
179
|
-
|
180
|
-
|
175
|
+
class OrMatchPattern < MatchPattern
|
176
|
+
def initialize(patterns)
|
177
|
+
@patterns = patterns
|
178
|
+
end
|
181
179
|
|
182
|
-
|
183
|
-
|
180
|
+
def match(str)
|
181
|
+
@patterns.any? {|pattern| pattern.match(str) }
|
182
|
+
end
|
184
183
|
end
|
185
184
|
end
|
186
|
-
|
187
|
-
|
188
|
-
end
|
189
|
-
|
data/lib/fluent/mixin.rb
CHANGED
@@ -16,207 +16,203 @@
|
|
16
16
|
# limitations under the License.
|
17
17
|
#
|
18
18
|
module Fluent
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
19
|
+
class TimeFormatter
|
20
|
+
def initialize(format, localtime)
|
21
|
+
@tc1 = 0
|
22
|
+
@tc1_str = nil
|
23
|
+
@tc2 = 0
|
24
|
+
@tc2_str = nil
|
25
|
+
|
26
|
+
if format
|
27
|
+
if localtime
|
28
|
+
define_singleton_method(:format_nocache) {|time|
|
29
|
+
Time.at(time).strftime(format)
|
30
|
+
}
|
31
|
+
else
|
32
|
+
define_singleton_method(:format_nocache) {|time|
|
33
|
+
Time.at(time).utc.strftime(format)
|
34
|
+
}
|
35
|
+
end
|
33
36
|
else
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
+
if localtime
|
38
|
+
define_singleton_method(:format_nocache) {|time|
|
39
|
+
Time.at(time).iso8601
|
40
|
+
}
|
41
|
+
else
|
42
|
+
define_singleton_method(:format_nocache) {|time|
|
43
|
+
Time.at(time).utc.iso8601
|
44
|
+
}
|
45
|
+
end
|
37
46
|
end
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
47
|
+
end
|
48
|
+
|
49
|
+
def format(time)
|
50
|
+
if @tc1 == time
|
51
|
+
return @tc1_str
|
52
|
+
elsif @tc2 == time
|
53
|
+
return @tc2_str
|
43
54
|
else
|
44
|
-
|
45
|
-
|
46
|
-
|
55
|
+
str = format_nocache(time)
|
56
|
+
if @tc1 < @tc2
|
57
|
+
@tc1 = time
|
58
|
+
@tc1_str = str
|
59
|
+
else
|
60
|
+
@tc2 = time
|
61
|
+
@tc2_str = str
|
62
|
+
end
|
63
|
+
return str
|
47
64
|
end
|
48
65
|
end
|
49
|
-
end
|
50
66
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
if @tc1 < @tc2
|
59
|
-
@tc1 = time
|
60
|
-
@tc1_str = str
|
61
|
-
else
|
62
|
-
@tc2 = time
|
63
|
-
@tc2_str = str
|
67
|
+
def format_nocache(time)
|
68
|
+
# will be overridden in initialize
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.configure(conf)
|
72
|
+
if time_format = conf['time_format']
|
73
|
+
@time_format = time_format
|
64
74
|
end
|
65
|
-
|
75
|
+
|
76
|
+
if localtime = conf['localtime']
|
77
|
+
@localtime = true
|
78
|
+
elsif utc = conf['utc']
|
79
|
+
@localtime = false
|
80
|
+
end
|
81
|
+
|
82
|
+
@timef = new(@time_format, @localtime)
|
66
83
|
end
|
67
84
|
end
|
68
85
|
|
69
|
-
def format_nocache(time)
|
70
|
-
# will be overridden in initialize
|
71
|
-
end
|
72
86
|
|
73
|
-
|
74
|
-
|
75
|
-
@time_format = time_format
|
87
|
+
module RecordFilterMixin
|
88
|
+
def filter_record(tag, time, record)
|
76
89
|
end
|
77
90
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
91
|
+
def format_stream(tag, es)
|
92
|
+
out = ''
|
93
|
+
es.each {|time,record|
|
94
|
+
tag_temp = tag.dup
|
95
|
+
filter_record(tag_temp, time, record)
|
96
|
+
out << format(tag_temp, time, record)
|
97
|
+
}
|
98
|
+
out
|
82
99
|
end
|
83
|
-
|
84
|
-
@timef = new(@time_format, @localtime)
|
85
100
|
end
|
86
|
-
end
|
87
|
-
|
88
101
|
|
89
|
-
module
|
90
|
-
|
91
|
-
end
|
102
|
+
module HandleTagNameMixin
|
103
|
+
include RecordFilterMixin
|
92
104
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
}
|
100
|
-
out
|
101
|
-
end
|
102
|
-
end
|
105
|
+
attr_accessor :remove_tag_prefix, :remove_tag_suffix, :add_tag_prefix, :add_tag_suffix
|
106
|
+
def configure(conf)
|
107
|
+
super
|
108
|
+
if remove_tag_prefix = conf['remove_tag_prefix']
|
109
|
+
@remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
|
110
|
+
end
|
103
111
|
|
104
|
-
|
105
|
-
|
112
|
+
if remove_tag_suffix = conf['remove_tag_suffix']
|
113
|
+
@remove_tag_suffix = Regexp.new(Regexp.escape(remove_tag_suffix) + '$')
|
114
|
+
end
|
106
115
|
|
107
|
-
|
108
|
-
|
109
|
-
super
|
110
|
-
if remove_tag_prefix = conf['remove_tag_prefix']
|
111
|
-
@remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
|
116
|
+
@add_tag_prefix = conf['add_tag_prefix']
|
117
|
+
@add_tag_suffix = conf['add_tag_suffix']
|
112
118
|
end
|
113
119
|
|
114
|
-
|
115
|
-
|
120
|
+
def filter_record(tag, time, record)
|
121
|
+
tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
|
122
|
+
tag.sub!(@remove_tag_suffix, '') if @remove_tag_suffix
|
123
|
+
tag.insert(0, @add_tag_prefix) if @add_tag_prefix
|
124
|
+
tag << @add_tag_suffix if @add_tag_suffix
|
125
|
+
super(tag, time, record)
|
116
126
|
end
|
117
|
-
|
118
|
-
@add_tag_prefix = conf['add_tag_prefix']
|
119
|
-
@add_tag_suffix = conf['add_tag_suffix']
|
120
127
|
end
|
121
128
|
|
122
|
-
|
123
|
-
|
124
|
-
tag.sub!(@remove_tag_suffix, '') if @remove_tag_suffix
|
125
|
-
tag.insert(0, @add_tag_prefix) if @add_tag_prefix
|
126
|
-
tag << @add_tag_suffix if @add_tag_suffix
|
127
|
-
super(tag, time, record)
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
module SetTimeKeyMixin
|
132
|
-
include RecordFilterMixin
|
129
|
+
module SetTimeKeyMixin
|
130
|
+
include RecordFilterMixin
|
133
131
|
|
134
|
-
|
132
|
+
attr_accessor :include_time_key, :time_key, :localtime
|
135
133
|
|
136
|
-
|
137
|
-
|
134
|
+
def configure(conf)
|
135
|
+
super
|
138
136
|
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
137
|
+
if s = conf['include_time_key']
|
138
|
+
b = Config.bool_value(s)
|
139
|
+
if s.empty?
|
140
|
+
b = true
|
141
|
+
elsif b == nil
|
142
|
+
raise ConfigError, "Invalid boolean expression '#{s}' for include_time_key parameter"
|
143
|
+
end
|
144
|
+
@include_time_key = b
|
145
145
|
end
|
146
|
-
@include_time_key = b
|
147
|
-
end
|
148
146
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
147
|
+
if @include_time_key
|
148
|
+
if time_key = conf['time_key']
|
149
|
+
@time_key = time_key
|
150
|
+
end
|
151
|
+
unless @time_key
|
152
|
+
@time_key = 'time'
|
153
|
+
end
|
156
154
|
|
157
|
-
|
158
|
-
|
159
|
-
|
155
|
+
if time_format = conf['time_format']
|
156
|
+
@time_format = time_format
|
157
|
+
end
|
160
158
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
159
|
+
if localtime = conf['localtime']
|
160
|
+
@localtime = true
|
161
|
+
elsif utc = conf['utc']
|
162
|
+
@localtime = false
|
163
|
+
end
|
166
164
|
|
167
|
-
|
165
|
+
@timef = TimeFormatter.new(@time_format, @localtime)
|
168
166
|
|
169
|
-
|
170
|
-
|
167
|
+
else
|
168
|
+
@include_time_key = false
|
169
|
+
end
|
171
170
|
end
|
172
|
-
end
|
173
171
|
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
172
|
+
def filter_record(tag, time, record)
|
173
|
+
super
|
174
|
+
if @include_time_key
|
175
|
+
record[@time_key] = @timef.format(time)
|
176
|
+
end
|
178
177
|
end
|
179
178
|
end
|
180
|
-
end
|
181
179
|
|
182
|
-
module SetTagKeyMixin
|
183
|
-
|
180
|
+
module SetTagKeyMixin
|
181
|
+
include RecordFilterMixin
|
184
182
|
|
185
|
-
|
183
|
+
attr_accessor :include_tag_key, :tag_key
|
186
184
|
|
187
|
-
|
188
|
-
|
185
|
+
def configure(conf)
|
186
|
+
super
|
189
187
|
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
188
|
+
if s = conf['include_tag_key']
|
189
|
+
b = Config.bool_value(s)
|
190
|
+
if s.empty?
|
191
|
+
b = true
|
192
|
+
elsif b == nil
|
193
|
+
raise ConfigError, "Invalid boolean expression '#{s}' for include_tag_key parameter"
|
194
|
+
end
|
195
|
+
@include_tag_key = b
|
196
196
|
end
|
197
|
-
@include_tag_key = b
|
198
|
-
end
|
199
197
|
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
198
|
+
if @include_tag_key
|
199
|
+
if tag_key = conf['tag_key']
|
200
|
+
@tag_key = tag_key
|
201
|
+
end
|
202
|
+
unless @tag_key
|
203
|
+
@tag_key = 'tag'
|
204
|
+
end
|
207
205
|
|
208
|
-
|
209
|
-
|
206
|
+
else
|
207
|
+
@include_tag_key = false
|
208
|
+
end
|
210
209
|
end
|
211
|
-
end
|
212
210
|
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
211
|
+
def filter_record(tag, time, record)
|
212
|
+
super
|
213
|
+
if @include_tag_key
|
214
|
+
record[@tag_key] = tag
|
215
|
+
end
|
217
216
|
end
|
218
217
|
end
|
219
218
|
end
|
220
|
-
|
221
|
-
|
222
|
-
end
|