fluentd 0.12.0.pre.3 → 0.12.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/fluent.conf +26 -16
- data/lib/fluent/agent.rb +7 -5
- data/lib/fluent/config.rb +1 -8
- data/lib/fluent/config/v1_parser.rb +2 -1
- data/lib/fluent/event_router.rb +1 -8
- data/lib/fluent/filter.rb +16 -0
- data/lib/fluent/formatter.rb +19 -18
- data/lib/fluent/label.rb +16 -0
- data/lib/fluent/output.rb +2 -2
- data/lib/fluent/parser.rb +57 -65
- data/lib/fluent/plugin/buf_file.rb +2 -2
- data/lib/fluent/plugin/filter_grep.rb +16 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +16 -0
- data/lib/fluent/plugin/in_http.rb +17 -5
- data/lib/fluent/plugin/in_monitor_agent.rb +52 -19
- data/lib/fluent/plugin/in_syslog.rb +2 -2
- data/lib/fluent/plugin/in_tail.rb +3 -3
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_roundrobin.rb +1 -1
- data/lib/fluent/plugin/socket_util.rb +1 -1
- data/lib/fluent/root_agent.rb +7 -12
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/test_in_http.rb +28 -8
- data/test/test_formatter.rb +12 -0
- data/test/test_output.rb +16 -0
- data/test/test_parser.rb +119 -75
- data/test/test_plugin_classes.rb +38 -0
- data/test/test_root_agent.rb +135 -0
- metadata +6 -5
- data/Gemfile.cool.io.1.1.1 +0 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 76237f968249345377ad1587cc876fe7783d5f74
|
4
|
+
data.tar.gz: a61a06b92b56a1ab55c729d1e3500a8856d7aeef
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 145f3e187ed6e48d46ffc08d5c6800c66194b1c726f6d8dea36504856890dc13adde304c7fab420a8055f86a740dd58832a4269cebbf03125e02bbbb2c9e45d4
|
7
|
+
data.tar.gz: a050b3b03cf3d1395caab0f8ea36098edec10aafb826812a40e9494cc645d51bf97d916f70e71bec848389f44d64f7f828f81aeb42e06894878a36b872d6504c
|
data/fluent.conf
CHANGED
@@ -1,26 +1,31 @@
|
|
1
|
+
# In v1 configuration, type and id are @ prefix parameters.
|
2
|
+
# @type and @id are recommended. type and id are still available for backward compatibility
|
1
3
|
|
2
4
|
## built-in TCP input
|
3
5
|
## $ echo <json> | fluent-cat <tag>
|
4
6
|
<source>
|
5
|
-
type forward
|
7
|
+
@type forward
|
8
|
+
@id forward_input
|
6
9
|
</source>
|
7
10
|
|
8
11
|
## built-in UNIX socket input
|
9
12
|
#<source>
|
10
|
-
# type unix
|
13
|
+
# @type unix
|
11
14
|
#</source>
|
12
15
|
|
13
16
|
# HTTP input
|
14
17
|
# http://localhost:8888/<tag>?json=<json>
|
15
18
|
<source>
|
16
|
-
type http
|
19
|
+
@type http
|
20
|
+
@id http_input
|
21
|
+
|
17
22
|
port 8888
|
18
23
|
</source>
|
19
24
|
|
20
25
|
## File input
|
21
26
|
## read apache logs with tag=apache.access
|
22
27
|
#<source>
|
23
|
-
# type tail
|
28
|
+
# @type tail
|
24
29
|
# format apache
|
25
30
|
# path /var/log/httpd-access.log
|
26
31
|
# tag apache.access
|
@@ -31,32 +36,38 @@
|
|
31
36
|
# http://localhost:24220/api/plugins?type=TYPE
|
32
37
|
# http://localhost:24220/api/plugins?tag=MYTAG
|
33
38
|
<source>
|
34
|
-
type monitor_agent
|
39
|
+
@type monitor_agent
|
40
|
+
@id monitor_agent_input
|
41
|
+
|
35
42
|
port 24220
|
36
43
|
</source>
|
37
44
|
|
38
45
|
# Listen DRb for debug
|
39
46
|
<source>
|
40
|
-
type debug_agent
|
47
|
+
@type debug_agent
|
48
|
+
@id debug_agent_input
|
49
|
+
|
41
50
|
bind 127.0.0.1
|
42
51
|
port 24230
|
43
52
|
</source>
|
44
53
|
|
45
|
-
|
46
54
|
## match tag=apache.access and write to file
|
47
55
|
#<match apache.access>
|
48
|
-
# type file
|
56
|
+
# @type file
|
49
57
|
# path /var/log/fluent/access
|
50
58
|
#</match>
|
51
59
|
|
52
60
|
## match tag=debug.** and dump to console
|
53
61
|
<match debug.**>
|
54
|
-
type stdout
|
62
|
+
@type stdout
|
63
|
+
@id stdout_output
|
55
64
|
</match>
|
56
65
|
|
57
66
|
# match tag=system.** and forward to another fluent server
|
58
67
|
<match system.**>
|
59
|
-
type forward
|
68
|
+
@type forward
|
69
|
+
@id forward_output
|
70
|
+
|
60
71
|
<server>
|
61
72
|
host 192.168.0.11
|
62
73
|
</server>
|
@@ -69,9 +80,9 @@
|
|
69
80
|
|
70
81
|
## match tag=myapp.** and forward and write to file
|
71
82
|
#<match myapp.**>
|
72
|
-
# type copy
|
83
|
+
# @type copy
|
73
84
|
# <store>
|
74
|
-
# type forward
|
85
|
+
# @type forward
|
75
86
|
# buffer_type file
|
76
87
|
# buffer_path /var/log/fluent/myapp-forward
|
77
88
|
# retry_limit 50
|
@@ -81,20 +92,19 @@
|
|
81
92
|
# </server>
|
82
93
|
# </store>
|
83
94
|
# <store>
|
84
|
-
# type file
|
95
|
+
# @type file
|
85
96
|
# path /var/log/fluent/myapp
|
86
97
|
# </store>
|
87
98
|
#</match>
|
88
99
|
|
89
100
|
## match fluent's internal events
|
90
101
|
#<match fluent.**>
|
91
|
-
# type null
|
102
|
+
# @type null
|
92
103
|
#</match>
|
93
104
|
|
94
105
|
## match not matched logs and write to file
|
95
106
|
#<match **>
|
96
|
-
# type file
|
107
|
+
# @type file
|
97
108
|
# path /var/log/fluent/else
|
98
109
|
# compress gz
|
99
110
|
#</match>
|
100
|
-
|
data/lib/fluent/agent.rb
CHANGED
@@ -27,7 +27,6 @@ module Fluent
|
|
27
27
|
include Configurable
|
28
28
|
|
29
29
|
def initialize(opts = {})
|
30
|
-
# initialize Configurable
|
31
30
|
super()
|
32
31
|
|
33
32
|
@context = nil
|
@@ -43,6 +42,7 @@ module Fluent
|
|
43
42
|
|
44
43
|
attr_reader :log
|
45
44
|
attr_reader :outputs
|
45
|
+
attr_reader :filters
|
46
46
|
attr_reader :context
|
47
47
|
attr_reader :event_router
|
48
48
|
attr_reader :error_collector
|
@@ -53,7 +53,7 @@ module Fluent
|
|
53
53
|
# initialize <match> and <filter> elements
|
54
54
|
conf.elements.select { |e| e.name == 'filter' || e.name == 'match' }.each { |e|
|
55
55
|
pattern = e.arg.empty? ? '**' : e.arg
|
56
|
-
type = e['type']
|
56
|
+
type = e['@type'] || e['type']
|
57
57
|
if e.name == 'filter'
|
58
58
|
add_filter(type, pattern, e)
|
59
59
|
else
|
@@ -62,7 +62,6 @@ module Fluent
|
|
62
62
|
}
|
63
63
|
end
|
64
64
|
|
65
|
-
# agent API called by Worker
|
66
65
|
def start
|
67
66
|
@outputs.each { |o|
|
68
67
|
o.start
|
@@ -75,7 +74,6 @@ module Fluent
|
|
75
74
|
}
|
76
75
|
end
|
77
76
|
|
78
|
-
# agent API called by Worker
|
79
77
|
def shutdown
|
80
78
|
@started_filters.map { |f|
|
81
79
|
Thread.new do
|
@@ -145,7 +143,11 @@ module Fluent
|
|
145
143
|
filter
|
146
144
|
end
|
147
145
|
|
148
|
-
|
146
|
+
# For handling invalid record
|
147
|
+
def emit_error_event(tag, time, record, error)
|
148
|
+
end
|
149
|
+
|
150
|
+
def handle_emits_error(tag, es, error)
|
149
151
|
end
|
150
152
|
|
151
153
|
class NoMatchMatch
|
data/lib/fluent/config.rb
CHANGED
@@ -43,17 +43,10 @@ module Fluent
|
|
43
43
|
|
44
44
|
module PluginId
|
45
45
|
def configure(conf)
|
46
|
-
@id = conf['id']
|
46
|
+
@id = conf['@id'] || conf['id']
|
47
47
|
super
|
48
48
|
end
|
49
49
|
|
50
|
-
def require_id
|
51
|
-
unless @id
|
52
|
-
raise ConfigError, "'id' parameter is required"
|
53
|
-
end
|
54
|
-
@id
|
55
|
-
end
|
56
|
-
|
57
50
|
def plugin_id
|
58
51
|
@id ? @id : "object:#{object_id.to_s(16)}"
|
59
52
|
end
|
@@ -51,6 +51,7 @@ module Fluent
|
|
51
51
|
end
|
52
52
|
|
53
53
|
ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
|
54
|
+
RESERVED_PARAMS = %W(@type @id @label)
|
54
55
|
|
55
56
|
def parse_element(root_element, elem_name, attrs = {}, elems = [])
|
56
57
|
while true
|
@@ -108,7 +109,7 @@ module Fluent
|
|
108
109
|
else
|
109
110
|
if k == '@include'
|
110
111
|
parse_include(attrs, elems)
|
111
|
-
elsif k
|
112
|
+
elsif RESERVED_PARAMS.include?(k)
|
112
113
|
v = parse_literal
|
113
114
|
unless line_end
|
114
115
|
parse_error! "expected end of line"
|
data/lib/fluent/event_router.rb
CHANGED
@@ -42,17 +42,10 @@ module Fluent
|
|
42
42
|
@match_rules = []
|
43
43
|
@match_cache = MatchCache.new
|
44
44
|
@default_collector = default_collector
|
45
|
-
@emit_error_handler = emit_error_handler
|
45
|
+
@emit_error_handler = emit_error_handler
|
46
46
|
@chain = NullOutputChain.instance
|
47
47
|
end
|
48
48
|
|
49
|
-
# Agent implements EmitErrorHandler. See 'fluentd/agent.rb'
|
50
|
-
class RaiseEmitErrorHandler
|
51
|
-
def handle_emits_error(tag, es, error)
|
52
|
-
raise error
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
49
|
attr_accessor :default_collector
|
57
50
|
attr_accessor :emit_error_handler
|
58
51
|
|
data/lib/fluent/filter.rb
CHANGED
@@ -1,3 +1,19 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
1
17
|
module Fluent
|
2
18
|
class Filter
|
3
19
|
include Configurable
|
data/lib/fluent/formatter.rb
CHANGED
@@ -17,6 +17,18 @@
|
|
17
17
|
module Fluent
|
18
18
|
require 'fluent/registry'
|
19
19
|
|
20
|
+
class Formatter
|
21
|
+
include Configurable
|
22
|
+
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
end
|
26
|
+
|
27
|
+
def format(tag, time, record)
|
28
|
+
raise NotImplementedError, "Implement this method in child class"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
20
32
|
module TextFormatter
|
21
33
|
module HandleTagAndTimeMixin
|
22
34
|
def self.included(klass)
|
@@ -50,8 +62,7 @@ module Fluent
|
|
50
62
|
end
|
51
63
|
end
|
52
64
|
|
53
|
-
class OutFileFormatter
|
54
|
-
include Configurable
|
65
|
+
class OutFileFormatter < Formatter
|
55
66
|
include HandleTagAndTimeMixin
|
56
67
|
|
57
68
|
config_param :output_time, :bool, :default => true
|
@@ -64,10 +75,6 @@ module Fluent
|
|
64
75
|
end
|
65
76
|
end
|
66
77
|
|
67
|
-
def configure(conf)
|
68
|
-
super
|
69
|
-
end
|
70
|
-
|
71
78
|
def format(tag, time, record)
|
72
79
|
filter_record(tag, time, record)
|
73
80
|
header = ''
|
@@ -104,8 +111,7 @@ module Fluent
|
|
104
111
|
end
|
105
112
|
end
|
106
113
|
|
107
|
-
class JSONFormatter
|
108
|
-
include Configurable
|
114
|
+
class JSONFormatter < Formatter
|
109
115
|
include HandleTagAndTimeMixin
|
110
116
|
include StructuredFormatMixin
|
111
117
|
|
@@ -114,8 +120,7 @@ module Fluent
|
|
114
120
|
end
|
115
121
|
end
|
116
122
|
|
117
|
-
class MessagePackFormatter
|
118
|
-
include Configurable
|
123
|
+
class MessagePackFormatter < Formatter
|
119
124
|
include HandleTagAndTimeMixin
|
120
125
|
include StructuredFormatMixin
|
121
126
|
|
@@ -124,8 +129,7 @@ module Fluent
|
|
124
129
|
end
|
125
130
|
end
|
126
131
|
|
127
|
-
class LabeledTSVFormatter
|
128
|
-
include Configurable
|
132
|
+
class LabeledTSVFormatter < Formatter
|
129
133
|
include HandleTagAndTimeMixin
|
130
134
|
|
131
135
|
config_param :delimiter, :string, :default => "\t"
|
@@ -142,8 +146,7 @@ module Fluent
|
|
142
146
|
end
|
143
147
|
end
|
144
148
|
|
145
|
-
class CsvFormatter
|
146
|
-
include Configurable
|
149
|
+
class CsvFormatter < Formatter
|
147
150
|
include HandleTagAndTimeMixin
|
148
151
|
|
149
152
|
config_param :delimiter, :default => ',' do |val|
|
@@ -173,9 +176,7 @@ module Fluent
|
|
173
176
|
end
|
174
177
|
end
|
175
178
|
|
176
|
-
class SingleValueFormatter
|
177
|
-
include Configurable
|
178
|
-
|
179
|
+
class SingleValueFormatter < Formatter
|
179
180
|
config_param :message_key, :string, :default => 'message'
|
180
181
|
config_param :add_newline, :bool, :default => true
|
181
182
|
|
@@ -186,7 +187,7 @@ module Fluent
|
|
186
187
|
end
|
187
188
|
end
|
188
189
|
|
189
|
-
class ProcWrappedFormatter
|
190
|
+
class ProcWrappedFormatter < Formatter
|
190
191
|
def initialize(proc)
|
191
192
|
@proc = proc
|
192
193
|
end
|
data/lib/fluent/label.rb
CHANGED
@@ -1,3 +1,19 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
1
17
|
module Fluent
|
2
18
|
require 'fluent/agent'
|
3
19
|
|
data/lib/fluent/output.rb
CHANGED
@@ -208,7 +208,7 @@ module Fluent
|
|
208
208
|
}
|
209
209
|
|
210
210
|
if sconf = conf.elements.select {|e| e.name == 'secondary' }.first
|
211
|
-
type = sconf['type'] || conf['type']
|
211
|
+
type = sconf['@type'] || conf['@type'] || sconf['type'] || conf['type']
|
212
212
|
@secondary = Plugin.new_output(type)
|
213
213
|
@secondary.configure(sconf)
|
214
214
|
|
@@ -401,7 +401,7 @@ module Fluent
|
|
401
401
|
# secondary retry
|
402
402
|
@retry_wait * (2 ** (@num_errors - 2 - @retry_limit))
|
403
403
|
end
|
404
|
-
retry_wait = wait + (rand * (wait / 4.0) - (wait / 8.0))
|
404
|
+
retry_wait = wait.finite? ? wait + (rand * (wait / 4.0) - (wait / 8.0)) : wait
|
405
405
|
@max_retry_wait ? [retry_wait, @max_retry_wait].min : retry_wait
|
406
406
|
end
|
407
407
|
|
data/lib/fluent/parser.rb
CHANGED
@@ -17,10 +17,39 @@
|
|
17
17
|
module Fluent
|
18
18
|
require 'fluent/registry'
|
19
19
|
|
20
|
-
class
|
21
|
-
|
20
|
+
class ParserError < StandardError
|
21
|
+
end
|
22
|
+
|
23
|
+
class Parser
|
24
|
+
include Configurable
|
25
|
+
|
26
|
+
# SET false BEFORE CONFIGURE, to return nil when time not parsed
|
27
|
+
# 'configure()' may raise errors for unexpected configurations
|
28
|
+
attr_accessor :estimate_current_event
|
29
|
+
|
30
|
+
def initialize
|
31
|
+
super
|
32
|
+
@estimate_current_event = true
|
22
33
|
end
|
23
34
|
|
35
|
+
def configure(conf)
|
36
|
+
super
|
37
|
+
end
|
38
|
+
|
39
|
+
def parse(text)
|
40
|
+
raise NotImplementedError, "Implement this method in child class"
|
41
|
+
end
|
42
|
+
|
43
|
+
# Keep backward compatibility for existing plugins
|
44
|
+
def call(*a, &b)
|
45
|
+
parse(*a, &b)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
class TextParser
|
50
|
+
# Keep backward compatibility for existing plugins
|
51
|
+
ParserError = ::Fluent::ParserError
|
52
|
+
|
24
53
|
class TimeParser
|
25
54
|
def initialize(time_format)
|
26
55
|
@cache1_key = nil
|
@@ -128,16 +157,11 @@ module Fluent
|
|
128
157
|
end
|
129
158
|
end
|
130
159
|
|
131
|
-
class RegexpParser
|
132
|
-
include Configurable
|
160
|
+
class RegexpParser < Parser
|
133
161
|
include TypeConverter
|
134
162
|
|
135
163
|
config_param :time_format, :string, :default => nil
|
136
164
|
|
137
|
-
# SET false BEFORE CONFIGURE, to return nil when time not parsed
|
138
|
-
# 'configure()' may raise errors for unexpected configurations
|
139
|
-
attr_accessor :estimate_current_event
|
140
|
-
|
141
165
|
def initialize(regexp, conf={})
|
142
166
|
super()
|
143
167
|
@regexp = regexp
|
@@ -146,7 +170,6 @@ module Fluent
|
|
146
170
|
end
|
147
171
|
|
148
172
|
@time_parser = TimeParser.new(@time_format)
|
149
|
-
@estimate_current_event = true
|
150
173
|
@mutex = Mutex.new
|
151
174
|
end
|
152
175
|
|
@@ -159,7 +182,7 @@ module Fluent
|
|
159
182
|
{'format' => @regexp, 'time_format' => @time_format}
|
160
183
|
end
|
161
184
|
|
162
|
-
def
|
185
|
+
def parse(text)
|
163
186
|
m = @regexp.match(text)
|
164
187
|
unless m
|
165
188
|
if block_given?
|
@@ -200,21 +223,10 @@ module Fluent
|
|
200
223
|
end
|
201
224
|
end
|
202
225
|
|
203
|
-
class JSONParser
|
204
|
-
include Configurable
|
205
|
-
|
226
|
+
class JSONParser < Parser
|
206
227
|
config_param :time_key, :string, :default => 'time'
|
207
228
|
config_param :time_format, :string, :default => nil
|
208
229
|
|
209
|
-
# SET false BEFORE CONFIGURE, to return nil when time not parsed
|
210
|
-
# 'configure()' may raise errors for unexpected configurations
|
211
|
-
attr_accessor :estimate_current_event
|
212
|
-
|
213
|
-
def initialize
|
214
|
-
super
|
215
|
-
@estimate_current_event = true
|
216
|
-
end
|
217
|
-
|
218
230
|
def configure(conf)
|
219
231
|
super
|
220
232
|
|
@@ -224,7 +236,7 @@ module Fluent
|
|
224
236
|
end
|
225
237
|
end
|
226
238
|
|
227
|
-
def
|
239
|
+
def parse(text)
|
228
240
|
record = Yajl.load(text)
|
229
241
|
|
230
242
|
if value = record.delete(@time_key)
|
@@ -259,21 +271,13 @@ module Fluent
|
|
259
271
|
end
|
260
272
|
end
|
261
273
|
|
262
|
-
class ValuesParser
|
263
|
-
include Configurable
|
274
|
+
class ValuesParser < Parser
|
264
275
|
include TypeConverter
|
265
276
|
|
266
277
|
config_param :keys, :string
|
267
278
|
config_param :time_key, :string, :default => nil
|
268
279
|
config_param :time_format, :string, :default => nil
|
269
280
|
|
270
|
-
attr_accessor :estimate_current_event
|
271
|
-
|
272
|
-
def initialize
|
273
|
-
super
|
274
|
-
@estimate_current_event = true
|
275
|
-
end
|
276
|
-
|
277
281
|
def configure(conf)
|
278
282
|
super
|
279
283
|
|
@@ -330,11 +334,16 @@ module Fluent
|
|
330
334
|
class TSVParser < ValuesParser
|
331
335
|
config_param :delimiter, :string, :default => "\t"
|
332
336
|
|
333
|
-
def
|
337
|
+
def configure(conf)
|
338
|
+
super
|
339
|
+
@key_num = @keys.length
|
340
|
+
end
|
341
|
+
|
342
|
+
def parse(text)
|
334
343
|
if block_given?
|
335
|
-
yield values_map(text.split(@delimiter))
|
344
|
+
yield values_map(text.split(@delimiter, @key_num))
|
336
345
|
else
|
337
|
-
return values_map(text.split(@delimiter))
|
346
|
+
return values_map(text.split(@delimiter, @key_num))
|
338
347
|
end
|
339
348
|
end
|
340
349
|
end
|
@@ -349,7 +358,7 @@ module Fluent
|
|
349
358
|
super(conf)
|
350
359
|
end
|
351
360
|
|
352
|
-
def
|
361
|
+
def parse(text)
|
353
362
|
@keys = []
|
354
363
|
values = []
|
355
364
|
|
@@ -373,7 +382,7 @@ module Fluent
|
|
373
382
|
require 'csv'
|
374
383
|
end
|
375
384
|
|
376
|
-
def
|
385
|
+
def parse(text)
|
377
386
|
if block_given?
|
378
387
|
yield values_map(CSV.parse_line(text))
|
379
388
|
else
|
@@ -382,19 +391,10 @@ module Fluent
|
|
382
391
|
end
|
383
392
|
end
|
384
393
|
|
385
|
-
class NoneParser
|
386
|
-
include Configurable
|
387
|
-
|
394
|
+
class NoneParser < Parser
|
388
395
|
config_param :message_key, :string, :default => 'message'
|
389
396
|
|
390
|
-
|
391
|
-
|
392
|
-
def initialize
|
393
|
-
super
|
394
|
-
@estimate_current_event = true
|
395
|
-
end
|
396
|
-
|
397
|
-
def call(text)
|
397
|
+
def parse(text)
|
398
398
|
record = {}
|
399
399
|
record[@message_key] = text
|
400
400
|
time = @estimate_current_event ? Engine.now : nil
|
@@ -406,13 +406,12 @@ module Fluent
|
|
406
406
|
end
|
407
407
|
end
|
408
408
|
|
409
|
-
class ApacheParser
|
410
|
-
include Configurable
|
411
|
-
|
409
|
+
class ApacheParser < Parser
|
412
410
|
REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
|
413
411
|
TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
|
414
412
|
|
415
413
|
def initialize
|
414
|
+
super
|
416
415
|
@time_parser = TimeParser.new(TIME_FORMAT)
|
417
416
|
@mutex = Mutex.new
|
418
417
|
end
|
@@ -421,7 +420,7 @@ module Fluent
|
|
421
420
|
{'format' => REGEXP, 'time_format' => TIME_FORMAT}
|
422
421
|
end
|
423
422
|
|
424
|
-
def
|
423
|
+
def parse(text)
|
425
424
|
m = REGEXP.match(text)
|
426
425
|
unless m
|
427
426
|
if block_given?
|
@@ -475,9 +474,7 @@ module Fluent
|
|
475
474
|
end
|
476
475
|
end
|
477
476
|
|
478
|
-
class SyslogParser
|
479
|
-
include Configurable
|
480
|
-
|
477
|
+
class SyslogParser < Parser
|
481
478
|
# From existence TextParser pattern
|
482
479
|
REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
|
483
480
|
# From in_syslog default pattern
|
@@ -486,11 +483,8 @@ module Fluent
|
|
486
483
|
config_param :time_format, :string, :default => "%b %d %H:%M:%S"
|
487
484
|
config_param :with_priority, :bool, :default => false
|
488
485
|
|
489
|
-
attr_accessor :estimate_current_event
|
490
|
-
|
491
486
|
def initialize
|
492
487
|
super
|
493
|
-
@estimate_current_event = true
|
494
488
|
@mutex = Mutex.new
|
495
489
|
end
|
496
490
|
|
@@ -505,7 +499,7 @@ module Fluent
|
|
505
499
|
{'format' => @regexp, 'time_format' => @time_format}
|
506
500
|
end
|
507
501
|
|
508
|
-
def
|
502
|
+
def parse(text)
|
509
503
|
m = @regexp.match(text)
|
510
504
|
unless m
|
511
505
|
if block_given?
|
@@ -544,9 +538,7 @@ module Fluent
|
|
544
538
|
end
|
545
539
|
end
|
546
540
|
|
547
|
-
class MultilineParser
|
548
|
-
include Configurable
|
549
|
-
|
541
|
+
class MultilineParser < Parser
|
550
542
|
config_param :format_firstline, :string, :default => nil
|
551
543
|
|
552
544
|
FORMAT_MAX_NUM = 20
|
@@ -571,7 +563,7 @@ module Fluent
|
|
571
563
|
end
|
572
564
|
end
|
573
565
|
|
574
|
-
def
|
566
|
+
def parse(text, &block)
|
575
567
|
if block
|
576
568
|
@parser.call(text, &block)
|
577
569
|
else
|
@@ -716,9 +708,9 @@ module Fluent
|
|
716
708
|
|
717
709
|
def parse(text, &block)
|
718
710
|
if block
|
719
|
-
@parser.
|
711
|
+
@parser.parse(text, &block)
|
720
712
|
else # keep backward compatibility. Will be removed at v1
|
721
|
-
return @parser.
|
713
|
+
return @parser.parse(text)
|
722
714
|
end
|
723
715
|
end
|
724
716
|
end
|