fluentd 0.10.48 → 0.10.49
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/ChangeLog +12 -0
- data/lib/fluent/command/debug.rb +8 -0
- data/lib/fluent/config/v1_parser.rb +23 -12
- data/lib/fluent/formatter.rb +173 -0
- data/lib/fluent/load.rb +1 -0
- data/lib/fluent/plugin/in_http.rb +63 -21
- data/lib/fluent/plugin/in_tail.rb +5 -1
- data/lib/fluent/plugin/out_file.rb +33 -21
- data/lib/fluent/supervisor.rb +26 -23
- data/lib/fluent/version.rb +1 -1
- data/spec/config/config_parser_spec.rb +99 -1
- data/test/plugin/test_in_http.rb +90 -7
- data/test/plugin/test_in_tail.rb +35 -0
- data/test/plugin/test_out_file.rb +60 -6
- data/test/scripts/fluent/plugin/formatter_known.rb +5 -0
- data/test/test_formatter.rb +208 -0
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9d9ea1a003d545a18baf0c61a85fb216d183cb07
|
4
|
+
data.tar.gz: c3ccc3a4665acdf5726287ce6b29e2e6af5c5194
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cd8c5f25fc9ca7cf31ac5d0fb2de908cee79204deb11e821e31774b8aecc112557ebcc0f0d2ddc033fcb4b189e31b05ff9fbe1a07d04ad4630a2b39980038f8b
|
7
|
+
data.tar.gz: af93420f7ed26fdf703d11723aa27e76a43528b90b67879687e8bb5fa94bb9bba55fbad61dbeaa10dc2218de776cc39df3e13372204a50359008a4abcebdf6f3
|
data/ChangeLog
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
Release 0.10.49 - 2014/06/05
|
2
|
+
|
3
|
+
* in_http: Add format option to support various input data format
|
4
|
+
* in_http: Accept json / msgpack array in default format
|
5
|
+
* in_tail: Print warning message when file not exist with 'read_from_head true'
|
6
|
+
* out_file: Add append option to disable path increment
|
7
|
+
* out_file: Add format option to support various output data format
|
8
|
+
* config: Fix broken 'include' processing in V1 configuration. Now add @include
|
9
|
+
* fluentd-debug: Fix undefined method 'usage' error when invalid option passed
|
10
|
+
* supervisor: Fix incorrect --group option handling
|
11
|
+
* Add TextFormatter module for output plugins
|
12
|
+
|
1
13
|
Release 0.10.48 - 2014/05/18
|
2
14
|
|
3
15
|
* config: Add inspect method to Section for dumping the status
|
data/lib/fluent/command/debug.rb
CHANGED
@@ -36,6 +36,14 @@ op.on('-u', '--unix PATH', "use unix socket instead of tcp") {|b|
|
|
36
36
|
unix = b
|
37
37
|
}
|
38
38
|
|
39
|
+
(class<<self;self;end).module_eval do
|
40
|
+
define_method(:usage) do |msg|
|
41
|
+
puts op.to_s
|
42
|
+
puts "error: #{msg}" if msg
|
43
|
+
exit 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
39
47
|
begin
|
40
48
|
op.parse!(ARGV)
|
41
49
|
|
@@ -94,10 +94,11 @@ module Fluent
|
|
94
94
|
e_attrs, e_elems = parse_element(false, e_name)
|
95
95
|
elems << Element.new(e_name, e_arg, e_attrs, e_elems)
|
96
96
|
|
97
|
-
elsif root_element && skip(
|
98
|
-
|
99
|
-
|
100
|
-
|
97
|
+
elsif root_element && skip(/(\@include|include)#{SPACING}/)
|
98
|
+
if !prev_match.start_with?('@')
|
99
|
+
$log.warn "'include' is deprecated. Use '@include' instead"
|
100
|
+
end
|
101
|
+
parse_include(attrs, elems)
|
101
102
|
|
102
103
|
else
|
103
104
|
k = scan_string(SPACING)
|
@@ -105,11 +106,15 @@ module Fluent
|
|
105
106
|
if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
|
106
107
|
attrs[k] = ""
|
107
108
|
else
|
108
|
-
|
109
|
-
|
110
|
-
|
109
|
+
if k == '@include'
|
110
|
+
parse_include(attrs, elems)
|
111
|
+
else
|
112
|
+
v = parse_literal
|
113
|
+
unless line_end
|
114
|
+
parse_error! "expected end of line"
|
115
|
+
end
|
116
|
+
attrs[k] = v
|
111
117
|
end
|
112
|
-
attrs[k] = v
|
113
118
|
end
|
114
119
|
end
|
115
120
|
end
|
@@ -117,6 +122,12 @@ module Fluent
|
|
117
122
|
return attrs, elems
|
118
123
|
end
|
119
124
|
|
125
|
+
def parse_include(attrs, elems)
|
126
|
+
uri = scan_string(LINE_END)
|
127
|
+
eval_include(attrs, elems, uri)
|
128
|
+
line_end
|
129
|
+
end
|
130
|
+
|
120
131
|
def eval_include(attrs, elems, uri)
|
121
132
|
u = URI.parse(uri)
|
122
133
|
if u.scheme == 'file' || u.path == uri # file path
|
@@ -127,12 +138,12 @@ module Fluent
|
|
127
138
|
pattern = path
|
128
139
|
end
|
129
140
|
|
130
|
-
Dir.glob(pattern).each { |path|
|
141
|
+
Dir.glob(pattern).sort.each { |path|
|
131
142
|
basepath = File.dirname(path)
|
132
143
|
fname = File.basename(path)
|
133
144
|
data = File.read(path)
|
134
145
|
ss = StringScanner.new(data)
|
135
|
-
V1Parser.new(ss, basepath, fname, @eval_context).
|
146
|
+
V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
|
136
147
|
}
|
137
148
|
|
138
149
|
else
|
@@ -140,9 +151,9 @@ module Fluent
|
|
140
151
|
fname = path
|
141
152
|
require 'open-uri'
|
142
153
|
data = nil
|
143
|
-
open(uri) { |f|
|
154
|
+
open(uri) { |f| data = f.read }
|
144
155
|
ss = StringScanner.new(data)
|
145
|
-
V1Parser.new(ss, basepath, fname, @eval_context).
|
156
|
+
V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
|
146
157
|
end
|
147
158
|
|
148
159
|
rescue SystemCallError => e
|
@@ -0,0 +1,173 @@
|
|
1
|
+
#
|
2
|
+
# Fluent
|
3
|
+
#
|
4
|
+
# Copyright (C) 2014 Fluentd project
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
module Fluent
|
19
|
+
require 'fluent/registry'
|
20
|
+
|
21
|
+
module TextFormatter
|
22
|
+
module HandleTagAndTimeMixin
|
23
|
+
def self.included(klass)
|
24
|
+
klass.instance_eval {
|
25
|
+
config_param :include_time_key, :bool, :default => false
|
26
|
+
config_param :time_key, :string, :default => 'time'
|
27
|
+
config_param :time_format, :string, :default => nil
|
28
|
+
config_param :include_tag_key, :bool, :default => false
|
29
|
+
config_param :tag_key, :string, :default => 'tag'
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
def configure(conf)
|
34
|
+
super
|
35
|
+
|
36
|
+
if conf['utc']
|
37
|
+
@localtime = false
|
38
|
+
elsif conf['localtime']
|
39
|
+
@localtime = true
|
40
|
+
end
|
41
|
+
@timef = TimeFormatter.new(@time_format, @localtime)
|
42
|
+
end
|
43
|
+
|
44
|
+
def filter_record(tag, time, record)
|
45
|
+
if @include_tag_key
|
46
|
+
record[@tag_key] = tag
|
47
|
+
end
|
48
|
+
if @include_time_key
|
49
|
+
record[@time_key] = @timef.format(time)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
class OutFileFormatter
|
55
|
+
include Configurable
|
56
|
+
include HandleTagAndTimeMixin
|
57
|
+
|
58
|
+
config_param :output_time, :bool, :default => true
|
59
|
+
config_param :output_tag, :bool, :default => true
|
60
|
+
config_param :delimiter, :default => "\t" do |val|
|
61
|
+
case val
|
62
|
+
when /SPACE/i then ' '
|
63
|
+
when /COMMA/i then ','
|
64
|
+
else "\t"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def configure(conf)
|
69
|
+
super
|
70
|
+
end
|
71
|
+
|
72
|
+
def format(tag, time, record)
|
73
|
+
filter_record(tag, time, record)
|
74
|
+
header = ''
|
75
|
+
header << "#{@timef.format(time)}#{@delimiter}" if @output_time
|
76
|
+
header << "#{tag}#{@delimiter}" if @output_tag
|
77
|
+
"#{header}#{Yajl.dump(record)}\n"
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
class JSONFormatter
|
82
|
+
include Configurable
|
83
|
+
include HandleTagAndTimeMixin
|
84
|
+
|
85
|
+
config_param :time_as_epoch, :bool, :default => false
|
86
|
+
|
87
|
+
def configure(conf)
|
88
|
+
super
|
89
|
+
|
90
|
+
if @time_as_epoch
|
91
|
+
if @include_time_key
|
92
|
+
@include_time_key = false
|
93
|
+
else
|
94
|
+
$log.warn "include_time_key is false so ignore time_as_epoch"
|
95
|
+
@time_as_epoch = false
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def format(tag, time, record)
|
101
|
+
filter_record(tag, time, record)
|
102
|
+
record[@time_key] = time if @time_as_epoch
|
103
|
+
"#{Yajl.dump(record)}\n"
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
class LabeledTSVFormatter
|
108
|
+
include Configurable
|
109
|
+
include HandleTagAndTimeMixin
|
110
|
+
|
111
|
+
config_param :delimiter, :string, :default => "\t"
|
112
|
+
config_param :label_delimiter, :string, :default => ":"
|
113
|
+
|
114
|
+
def format(tag, time, record)
|
115
|
+
filter_record(tag, time, record)
|
116
|
+
formatted = record.inject('') { |result, pair|
|
117
|
+
result << @delimiter if result.length.nonzero?
|
118
|
+
result << "#{pair.first}#{@label_delimiter}#{pair.last}"
|
119
|
+
}
|
120
|
+
formatted << "\n"
|
121
|
+
formatted
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
class SingleValueFormatter
|
126
|
+
include Configurable
|
127
|
+
|
128
|
+
config_param :message_key, :string, :default => 'message'
|
129
|
+
|
130
|
+
def format(tag, time, record)
|
131
|
+
record[@message_key]
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
TEMPLATE_REGISTRY = Registry.new(:formatter_type, 'fluent/plugin/formatter_')
|
136
|
+
{
|
137
|
+
'out_file' => Proc.new { OutFileFormatter.new },
|
138
|
+
'json' => Proc.new { JSONFormatter.new },
|
139
|
+
'ltsv' => Proc.new { LabeledTSVFormatter.new },
|
140
|
+
'single_value' => Proc.new { SingleValueFormatter.new },
|
141
|
+
}.each { |name, factory|
|
142
|
+
TEMPLATE_REGISTRY.register(name, factory)
|
143
|
+
}
|
144
|
+
|
145
|
+
def self.register_template(name, factory_or_proc)
|
146
|
+
factory = if factory_or_proc.arity == 3
|
147
|
+
Proc.new { factory_or_proc }
|
148
|
+
else
|
149
|
+
factory_or_proc
|
150
|
+
end
|
151
|
+
|
152
|
+
TEMPLATE_REGISTRY.register(name, factory)
|
153
|
+
end
|
154
|
+
|
155
|
+
def self.create(conf)
|
156
|
+
format = conf['format']
|
157
|
+
if format.nil?
|
158
|
+
raise ConfigError, "'format' parameter is required"
|
159
|
+
end
|
160
|
+
|
161
|
+
# built-in template
|
162
|
+
begin
|
163
|
+
factory = TEMPLATE_REGISTRY.lookup(format)
|
164
|
+
rescue ConfigError => e
|
165
|
+
raise ConfigError, "unknown format: '#{format}'"
|
166
|
+
end
|
167
|
+
|
168
|
+
formatter = factory.call
|
169
|
+
formatter.configure(conf)
|
170
|
+
formatter
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
data/lib/fluent/load.rb
CHANGED
@@ -35,9 +35,22 @@ module Fluent
|
|
35
35
|
config_param :keepalive_timeout, :time, :default => 10 # TODO default
|
36
36
|
config_param :backlog, :integer, :default => nil
|
37
37
|
config_param :add_http_headers, :bool, :default => false
|
38
|
+
config_param :format, :string, :default => 'default'
|
38
39
|
|
39
40
|
def configure(conf)
|
40
41
|
super
|
42
|
+
|
43
|
+
m = if @format == 'default'
|
44
|
+
method(:parse_params_default)
|
45
|
+
else
|
46
|
+
parser = TextParser.new
|
47
|
+
parser.configure(conf)
|
48
|
+
@parser = parser
|
49
|
+
method(:parse_params_with_parser)
|
50
|
+
end
|
51
|
+
(class << self; self; end).module_eval do
|
52
|
+
define_method(:parse_params, m)
|
53
|
+
end
|
41
54
|
end
|
42
55
|
|
43
56
|
class KeepaliveManager < Coolio::TimerWatcher
|
@@ -79,7 +92,7 @@ module Fluent
|
|
79
92
|
super
|
80
93
|
@km = KeepaliveManager.new(@keepalive_timeout)
|
81
94
|
#@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
|
82
|
-
@lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, log)
|
95
|
+
@lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, @format, log)
|
83
96
|
@lsock.listen(@backlog) unless @backlog.nil?
|
84
97
|
|
85
98
|
@loop = Coolio::Loop.new
|
@@ -108,22 +121,13 @@ module Fluent
|
|
108
121
|
begin
|
109
122
|
path = path_info[1..-1] # remove /
|
110
123
|
tag = path.split('/').join('.')
|
111
|
-
|
112
|
-
if msgpack = params['msgpack']
|
113
|
-
record = MessagePack.unpack(msgpack)
|
114
|
-
|
115
|
-
elsif js = params['json']
|
116
|
-
record = JSON.parse(js)
|
117
|
-
|
118
|
-
else
|
119
|
-
raise "'json' or 'msgpack' parameter is required"
|
120
|
-
end
|
124
|
+
record_time, record = parse_params(params)
|
121
125
|
|
122
126
|
# Skip nil record
|
123
127
|
if record.nil?
|
124
128
|
return ["200 OK", {'Content-type'=>'text/plain'}, ""]
|
125
129
|
end
|
126
|
-
|
130
|
+
|
127
131
|
if @add_http_headers
|
128
132
|
params.each_pair { |k,v|
|
129
133
|
if k.start_with?("HTTP_")
|
@@ -132,19 +136,29 @@ module Fluent
|
|
132
136
|
}
|
133
137
|
end
|
134
138
|
|
135
|
-
time = params['time']
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
139
|
+
time = if param_time = params['time']
|
140
|
+
param_time = param_time.to_i
|
141
|
+
param_time.zero? ? Engine.now : param_time
|
142
|
+
else
|
143
|
+
record_time.nil? ? Engine.now : record_time
|
144
|
+
end
|
141
145
|
rescue
|
142
146
|
return ["400 Bad Request", {'Content-type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
|
143
147
|
end
|
144
148
|
|
145
149
|
# TODO server error
|
146
150
|
begin
|
147
|
-
|
151
|
+
# Support batched requests
|
152
|
+
if record.is_a?(Array)
|
153
|
+
mes = MultiEventStream.new
|
154
|
+
record.each do |single_record|
|
155
|
+
single_time = single_record.delete("time") || time
|
156
|
+
mes.add(single_time, single_record)
|
157
|
+
end
|
158
|
+
Engine.emit_stream(tag, mes)
|
159
|
+
else
|
160
|
+
Engine.emit(tag, time, record)
|
161
|
+
end
|
148
162
|
rescue
|
149
163
|
return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
|
150
164
|
end
|
@@ -152,14 +166,40 @@ module Fluent
|
|
152
166
|
return ["200 OK", {'Content-type'=>'text/plain'}, ""]
|
153
167
|
end
|
154
168
|
|
169
|
+
private
|
170
|
+
|
171
|
+
def parse_params_default(params)
|
172
|
+
record = if msgpack = params['msgpack']
|
173
|
+
MessagePack.unpack(msgpack)
|
174
|
+
elsif js = params['json']
|
175
|
+
JSON.parse(js)
|
176
|
+
else
|
177
|
+
raise "'json' or 'msgpack' parameter is required"
|
178
|
+
end
|
179
|
+
return nil, record
|
180
|
+
end
|
181
|
+
|
182
|
+
EVENT_RECORD_PARAMETER = '_event_record'
|
183
|
+
|
184
|
+
def parse_params_with_parser(params)
|
185
|
+
if content = params[EVENT_RECORD_PARAMETER]
|
186
|
+
time, record = @parser.parse(content)
|
187
|
+
raise "Received event is not #{@format}: #{content}" if record.nil?
|
188
|
+
return time, record
|
189
|
+
else
|
190
|
+
raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
155
194
|
class Handler < Coolio::Socket
|
156
|
-
def initialize(io, km, callback, body_size_limit, log)
|
195
|
+
def initialize(io, km, callback, body_size_limit, format, log)
|
157
196
|
super(io)
|
158
197
|
@km = km
|
159
198
|
@callback = callback
|
160
199
|
@body_size_limit = body_size_limit
|
161
200
|
@content_type = ""
|
162
201
|
@next_close = false
|
202
|
+
@format = format
|
163
203
|
@log = log
|
164
204
|
|
165
205
|
@idle = 0
|
@@ -250,7 +290,9 @@ module Fluent
|
|
250
290
|
uri = URI.parse(@parser.request_url)
|
251
291
|
params = WEBrick::HTTPUtils.parse_query(uri.query)
|
252
292
|
|
253
|
-
if @
|
293
|
+
if @format != 'default'
|
294
|
+
params[EVENT_RECORD_PARAMETER] = @body
|
295
|
+
elsif @content_type =~ /^application\/x-www-form-urlencoded/
|
254
296
|
params.update WEBrick::HTTPUtils.parse_query(@body)
|
255
297
|
elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
|
256
298
|
boundary = WEBrick::HTTPUtils.dequote($1)
|
@@ -140,7 +140,11 @@ module Fluent
|
|
140
140
|
if @pf
|
141
141
|
pe = @pf[path]
|
142
142
|
if @read_from_head && pe.read_inode.zero?
|
143
|
-
|
143
|
+
begin
|
144
|
+
pe.update(File::Stat.new(path).ino, 0)
|
145
|
+
rescue Errno::ENOENT
|
146
|
+
$log.warn "#{path} not found. Continuing without tailing it."
|
147
|
+
end
|
144
148
|
end
|
145
149
|
end
|
146
150
|
|
@@ -25,9 +25,8 @@ module Fluent
|
|
25
25
|
}
|
26
26
|
|
27
27
|
config_param :path, :string
|
28
|
-
|
29
|
-
config_param :
|
30
|
-
|
28
|
+
config_param :format, :string, :default => 'out_file'
|
29
|
+
config_param :append, :bool, :default => false
|
31
30
|
config_param :compress, :default => nil do |val|
|
32
31
|
c = SUPPORTED_COMPRESS[val]
|
33
32
|
unless c
|
@@ -35,7 +34,6 @@ module Fluent
|
|
35
34
|
end
|
36
35
|
c
|
37
36
|
end
|
38
|
-
|
39
37
|
config_param :symlink_path, :string, :default => nil
|
40
38
|
|
41
39
|
def initialize
|
@@ -64,29 +62,18 @@ module Fluent
|
|
64
62
|
|
65
63
|
super
|
66
64
|
|
67
|
-
|
65
|
+
conf['format'] = @format
|
66
|
+
@formatter = TextFormatter.create(conf)
|
68
67
|
|
69
68
|
@buffer.symlink_path = @symlink_path if @symlink_path
|
70
69
|
end
|
71
70
|
|
72
71
|
def format(tag, time, record)
|
73
|
-
|
74
|
-
"#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n"
|
72
|
+
@formatter.format(tag, time, record)
|
75
73
|
end
|
76
74
|
|
77
75
|
def write(chunk)
|
78
|
-
|
79
|
-
when nil
|
80
|
-
suffix = ''
|
81
|
-
when :gz
|
82
|
-
suffix = ".gz"
|
83
|
-
end
|
84
|
-
|
85
|
-
i = 0
|
86
|
-
begin
|
87
|
-
path = "#{@path_prefix}#{chunk.key}_#{i}#{@path_suffix}#{suffix}"
|
88
|
-
i += 1
|
89
|
-
end while File.exist?(path)
|
76
|
+
path = generate_path(chunk)
|
90
77
|
FileUtils.mkdir_p File.dirname(path)
|
91
78
|
|
92
79
|
case @compress
|
@@ -95,8 +82,10 @@ module Fluent
|
|
95
82
|
chunk.write_to(f)
|
96
83
|
}
|
97
84
|
when :gz
|
98
|
-
|
99
|
-
|
85
|
+
File.open(path, "a", DEFAULT_FILE_PERMISSION) {|f|
|
86
|
+
gz = Zlib::GzipWriter.new(f)
|
87
|
+
chunk.write_to(gz)
|
88
|
+
gz.close
|
100
89
|
}
|
101
90
|
end
|
102
91
|
|
@@ -106,5 +95,28 @@ module Fluent
|
|
106
95
|
def secondary_init(primary)
|
107
96
|
# don't warn even if primary.class is not FileOutput
|
108
97
|
end
|
98
|
+
|
99
|
+
private
|
100
|
+
|
101
|
+
def generate_path(chunk)
|
102
|
+
case @compress
|
103
|
+
when nil
|
104
|
+
suffix = ''
|
105
|
+
when :gz
|
106
|
+
suffix = ".gz"
|
107
|
+
end
|
108
|
+
|
109
|
+
if @append
|
110
|
+
"#{@path_prefix}#{chunk.key}#{@path_suffix}#{suffix}"
|
111
|
+
else
|
112
|
+
path = nil
|
113
|
+
i = 0
|
114
|
+
begin
|
115
|
+
path = "#{@path_prefix}#{chunk.key}_#{i}#{@path_suffix}#{suffix}"
|
116
|
+
i += 1
|
117
|
+
end while File.exist?(path)
|
118
|
+
path
|
119
|
+
end
|
120
|
+
end
|
109
121
|
end
|
110
122
|
end
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -18,9 +18,26 @@
|
|
18
18
|
|
19
19
|
require 'fluent/env'
|
20
20
|
require 'fluent/log'
|
21
|
+
require 'etc'
|
21
22
|
|
22
23
|
module Fluent
|
23
24
|
class Supervisor
|
25
|
+
def self.get_etc_passwd(user)
|
26
|
+
if user.to_i.to_s == user
|
27
|
+
Etc.getpwuid(user.to_i)
|
28
|
+
else
|
29
|
+
Etc.getpwnam(user)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.get_etc_group(group)
|
34
|
+
if group.to_i.to_s == group
|
35
|
+
Etc.getgrgid(group.to_i)
|
36
|
+
else
|
37
|
+
Etc.getgrnam(group)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
24
41
|
class LoggerInitializer
|
25
42
|
def initialize(path, level, chuser, chgroup, opts)
|
26
43
|
@path = path
|
@@ -34,8 +51,8 @@ module Fluent
|
|
34
51
|
if @path && @path != "-"
|
35
52
|
@io = File.open(@path, "a")
|
36
53
|
if @chuser || @chgroup
|
37
|
-
chuid = @chuser ?
|
38
|
-
chgid = @chgroup ?
|
54
|
+
chuid = @chuser ? Supervisor.get_etc_passwd(@chuser).uid : nil
|
55
|
+
chgid = @chgroup ? Supervisor.get_etc_group(@chgroup).gid : nil
|
39
56
|
File.chown(chuid, chgid, @path)
|
40
57
|
end
|
41
58
|
else
|
@@ -318,32 +335,18 @@ module Fluent
|
|
318
335
|
|
319
336
|
def change_privilege
|
320
337
|
if @chgroup
|
321
|
-
|
322
|
-
|
323
|
-
chgid = `id -g #{@chgroup}`.to_i
|
324
|
-
if $?.to_i != 0
|
325
|
-
exit 1
|
326
|
-
end
|
327
|
-
end
|
328
|
-
Process::GID.change_privilege(chgid)
|
338
|
+
etc_group = Supervisor.get_etc_group(@chgroup)
|
339
|
+
Process::GID.change_privilege(etc_group.gid)
|
329
340
|
end
|
330
341
|
|
331
342
|
if @chuser
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
exit 1
|
337
|
-
end
|
338
|
-
end
|
339
|
-
|
340
|
-
user_groups = `id -G #{@chuser}`.split.map(&:to_i)
|
341
|
-
if $?.to_i != 0
|
342
|
-
exit 1
|
343
|
-
end
|
343
|
+
etc_pw = Supervisor.get_etc_passwd(@chuser)
|
344
|
+
user_groups = [etc_pw.gid]
|
345
|
+
Etc.setgrent
|
346
|
+
Etc.group { |gr| user_groups << gr.gid if gr.mem.include?(etc_pw.name) } # emulate 'id -G'
|
344
347
|
|
345
348
|
Process.groups = Process.groups | user_groups
|
346
|
-
Process::UID.change_privilege(
|
349
|
+
Process::UID.change_privilege(etc_pw.uid)
|
347
350
|
end
|
348
351
|
end
|
349
352
|
|
data/lib/fluent/version.rb
CHANGED
@@ -177,7 +177,105 @@ describe Fluent::Config::V1Parser do
|
|
177
177
|
end
|
178
178
|
end
|
179
179
|
|
180
|
+
# port from test_config.rb
|
180
181
|
describe '@include parsing' do
|
181
|
-
#
|
182
|
+
TMP_DIR = File.dirname(__FILE__) + "/tmp/v1_config#{ENV['TEST_ENV_NUMBER']}"
|
183
|
+
|
184
|
+
def write_config(path, data)
|
185
|
+
FileUtils.mkdir_p(File.dirname(path))
|
186
|
+
File.open(path, "w") { |f| f.write data }
|
187
|
+
end
|
188
|
+
|
189
|
+
def prepare_config
|
190
|
+
write_config "#{TMP_DIR}/config_test_1.conf", %[
|
191
|
+
k1 root_config
|
192
|
+
include dir/config_test_2.conf #
|
193
|
+
@include #{TMP_DIR}/config_test_4.conf
|
194
|
+
include file://#{TMP_DIR}/config_test_5.conf
|
195
|
+
@include config.d/*.conf
|
196
|
+
]
|
197
|
+
write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
|
198
|
+
k2 relative_path_include
|
199
|
+
@include ../config_test_3.conf
|
200
|
+
]
|
201
|
+
write_config "#{TMP_DIR}/config_test_3.conf", %[
|
202
|
+
k3 relative_include_in_included_file
|
203
|
+
]
|
204
|
+
write_config "#{TMP_DIR}/config_test_4.conf", %[
|
205
|
+
k4 absolute_path_include
|
206
|
+
]
|
207
|
+
write_config "#{TMP_DIR}/config_test_5.conf", %[
|
208
|
+
k5 uri_include
|
209
|
+
]
|
210
|
+
write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
|
211
|
+
k6 wildcard_include_1
|
212
|
+
<elem1 name>
|
213
|
+
include normal_parameter
|
214
|
+
</elem1>
|
215
|
+
]
|
216
|
+
write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
|
217
|
+
k7 wildcard_include_2
|
218
|
+
]
|
219
|
+
write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
|
220
|
+
<elem2 name>
|
221
|
+
@include ../dir/config_test_9.conf
|
222
|
+
</elem2>
|
223
|
+
]
|
224
|
+
write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
|
225
|
+
k9 embeded
|
226
|
+
<elem3 name>
|
227
|
+
nested nested_value
|
228
|
+
include hoge
|
229
|
+
</elem3>
|
230
|
+
]
|
231
|
+
write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
|
232
|
+
k8 wildcard_include_3
|
233
|
+
<elem4 name>
|
234
|
+
include normal_parameter
|
235
|
+
</elem4>
|
236
|
+
]
|
237
|
+
end
|
238
|
+
|
239
|
+
it 'parses @include / include correctly' do
|
240
|
+
prepare_config
|
241
|
+
c = Fluent::Config.read("#{TMP_DIR}/config_test_1.conf", true)
|
242
|
+
expect(c['k1']).to eq('root_config')
|
243
|
+
expect(c['k2']).to eq('relative_path_include')
|
244
|
+
expect(c['k3']).to eq('relative_include_in_included_file')
|
245
|
+
expect(c['k4']).to eq('absolute_path_include')
|
246
|
+
expect(c['k5']).to eq('uri_include')
|
247
|
+
expect(c['k6']).to eq('wildcard_include_1')
|
248
|
+
expect(c['k7']).to eq('wildcard_include_2')
|
249
|
+
expect(c['k8']).to eq('wildcard_include_3')
|
250
|
+
expect(c.keys).to eq([
|
251
|
+
'k1',
|
252
|
+
'k2',
|
253
|
+
'k3',
|
254
|
+
'k4',
|
255
|
+
'k5',
|
256
|
+
'k8', # Because of the file name this comes first.
|
257
|
+
'k6',
|
258
|
+
'k7',
|
259
|
+
])
|
260
|
+
|
261
|
+
elem1 = c.elements.find { |e| e.name == 'elem1' }
|
262
|
+
expect(elem1).to be
|
263
|
+
expect(elem1.arg).to eq('name')
|
264
|
+
expect(elem1['include']).to eq('normal_parameter')
|
265
|
+
|
266
|
+
elem2 = c.elements.find { |e| e.name == 'elem2' }
|
267
|
+
expect(elem2).to be
|
268
|
+
expect(elem2.arg).to eq('name')
|
269
|
+
expect(elem2['k9']).to eq('embeded')
|
270
|
+
expect(elem2.has_key?('include')).to be(false)
|
271
|
+
|
272
|
+
elem3 = elem2.elements.find { |e| e.name == 'elem3' }
|
273
|
+
expect(elem3).to be
|
274
|
+
expect(elem3['nested']).to eq('nested_value')
|
275
|
+
expect(elem3['include']).to eq('hoge')
|
276
|
+
end
|
277
|
+
|
278
|
+
# TODO: Add uri based include spec
|
182
279
|
end
|
183
280
|
end
|
281
|
+
|
data/test/plugin/test_in_http.rb
CHANGED
@@ -65,6 +65,25 @@ class HttpInputTest < Test::Unit::TestCase
|
|
65
65
|
}
|
66
66
|
end
|
67
67
|
|
68
|
+
def test_multi_json
|
69
|
+
d = create_driver
|
70
|
+
|
71
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
72
|
+
|
73
|
+
events = [{"a"=>1},{"a"=>2}]
|
74
|
+
tag = "tag1"
|
75
|
+
|
76
|
+
events.each { |ev|
|
77
|
+
d.expect_emit tag, time, ev
|
78
|
+
}
|
79
|
+
|
80
|
+
d.run do
|
81
|
+
res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s})
|
82
|
+
assert_equal "200", res.code
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
|
68
87
|
def test_json_with_add_http_headers
|
69
88
|
d = create_driver(CONFIG + "add_http_headers true")
|
70
89
|
|
@@ -95,10 +114,7 @@ class HttpInputTest < Test::Unit::TestCase
|
|
95
114
|
|
96
115
|
d.run do
|
97
116
|
d.expected_emits.each {|tag,time,record|
|
98
|
-
|
99
|
-
req = Net::HTTP::Post.new("/#{tag}?time=#{time.to_s}", {"content-type"=>"application/json; charset=utf-8"})
|
100
|
-
req.body = record.to_json
|
101
|
-
res = http.request(req)
|
117
|
+
res = post("/#{tag}?time=#{time.to_s}", record.to_json, {"content-type"=>"application/json; charset=utf-8"})
|
102
118
|
assert_equal "200", res.code
|
103
119
|
}
|
104
120
|
end
|
@@ -120,10 +136,77 @@ class HttpInputTest < Test::Unit::TestCase
|
|
120
136
|
end
|
121
137
|
end
|
122
138
|
|
123
|
-
def
|
139
|
+
def test_multi_msgpack
|
140
|
+
d = create_driver
|
141
|
+
|
142
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
143
|
+
|
144
|
+
events = [{"a"=>1},{"a"=>2}]
|
145
|
+
tag = "tag1"
|
146
|
+
|
147
|
+
events.each { |ev|
|
148
|
+
d.expect_emit tag, time, ev
|
149
|
+
}
|
150
|
+
|
151
|
+
d.run do
|
152
|
+
res = post("/#{tag}", {"msgpack"=>events.to_msgpack, "time"=>time.to_s})
|
153
|
+
assert_equal "200", res.code
|
154
|
+
end
|
155
|
+
|
156
|
+
end
|
157
|
+
|
158
|
+
def test_with_regexp
|
159
|
+
d = create_driver(CONFIG + %[
|
160
|
+
format /^(?<field_1>\\d+):(?<field_2>\\w+)$/
|
161
|
+
types field_1:integer
|
162
|
+
])
|
163
|
+
|
164
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
165
|
+
|
166
|
+
d.expect_emit "tag1", time, {"field_1" => 1, "field_2" => 'str'}
|
167
|
+
d.expect_emit "tag2", time, {"field_1" => 2, "field_2" => 'str'}
|
168
|
+
|
169
|
+
d.run do
|
170
|
+
d.expected_emits.each { |tag, time, record|
|
171
|
+
body = record.map { |k, v|
|
172
|
+
v.to_s
|
173
|
+
}.join(':')
|
174
|
+
res = post("/#{tag}?time=#{time.to_s}", body)
|
175
|
+
assert_equal "200", res.code
|
176
|
+
}
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
def test_with_csv
|
181
|
+
require 'csv'
|
182
|
+
|
183
|
+
d = create_driver(CONFIG + %[
|
184
|
+
format csv
|
185
|
+
keys foo,bar
|
186
|
+
])
|
187
|
+
|
188
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
189
|
+
|
190
|
+
d.expect_emit "tag1", time, {"foo" => "1", "bar" => 'st"r'}
|
191
|
+
d.expect_emit "tag2", time, {"foo" => "2", "bar" => 'str'}
|
192
|
+
|
193
|
+
d.run do
|
194
|
+
d.expected_emits.each { |tag, time, record|
|
195
|
+
body = record.map { |k, v| v }.to_csv
|
196
|
+
res = post("/#{tag}?time=#{time.to_s}", body)
|
197
|
+
assert_equal "200", res.code
|
198
|
+
}
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def post(path, params, header = {})
|
124
203
|
http = Net::HTTP.new("127.0.0.1", PORT)
|
125
|
-
req = Net::HTTP::Post.new(path,
|
126
|
-
|
204
|
+
req = Net::HTTP::Post.new(path, header)
|
205
|
+
if params.is_a?(String)
|
206
|
+
req.body = params
|
207
|
+
else
|
208
|
+
req.set_form_data(params)
|
209
|
+
end
|
127
210
|
http.request(req)
|
128
211
|
end
|
129
212
|
|
data/test/plugin/test_in_tail.rb
CHANGED
@@ -337,4 +337,39 @@ class TailInputTest < Test::Unit::TestCase
|
|
337
337
|
plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
|
338
338
|
end
|
339
339
|
end
|
340
|
+
|
341
|
+
# Ensure that no fatal exception is raised when a file is missing and that
|
342
|
+
# files that do exist are still tailed as expected.
|
343
|
+
def test_missing_file
|
344
|
+
File.open("#{TMP_DIR}/tail.txt", "w") {|f|
|
345
|
+
f.puts "test1"
|
346
|
+
f.puts "test2"
|
347
|
+
}
|
348
|
+
|
349
|
+
# Try two different configs - one with read_from_head and one without,
|
350
|
+
# since their interactions with the filesystem differ.
|
351
|
+
config1 = %[
|
352
|
+
tag t1
|
353
|
+
path #{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt
|
354
|
+
format none
|
355
|
+
rotate_wait 2s
|
356
|
+
pos_file #{TMP_DIR}/tail.pos
|
357
|
+
]
|
358
|
+
config2 = config1 + ' read_from_head true'
|
359
|
+
[config1, config2].each do |config|
|
360
|
+
d = create_driver(config, false)
|
361
|
+
d.run do
|
362
|
+
sleep 1
|
363
|
+
File.open("#{TMP_DIR}/tail.txt", "a") {|f|
|
364
|
+
f.puts "test3"
|
365
|
+
f.puts "test4"
|
366
|
+
}
|
367
|
+
sleep 1
|
368
|
+
end
|
369
|
+
emits = d.emits
|
370
|
+
assert_equal(2, emits.length)
|
371
|
+
assert_equal({"message"=>"test3"}, emits[0][2])
|
372
|
+
assert_equal({"message"=>"test4"}, emits[1][2])
|
373
|
+
end
|
374
|
+
end
|
340
375
|
end
|
@@ -44,6 +44,24 @@ class FileOutputTest < Test::Unit::TestCase
|
|
44
44
|
d.run
|
45
45
|
end
|
46
46
|
|
47
|
+
def check_gzipped_result(path, expect)
|
48
|
+
# Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
|
49
|
+
# Following code from https://www.ruby-forum.com/topic/971591#979520
|
50
|
+
result = ''
|
51
|
+
File.open(path) { |io|
|
52
|
+
loop do
|
53
|
+
gzr = Zlib::GzipReader.new(io)
|
54
|
+
result << gzr.read
|
55
|
+
unused = gzr.unused
|
56
|
+
gzr.finish
|
57
|
+
break if unused.nil?
|
58
|
+
io.pos -= unused.length
|
59
|
+
end
|
60
|
+
}
|
61
|
+
|
62
|
+
assert_equal expect, result
|
63
|
+
end
|
64
|
+
|
47
65
|
def test_write
|
48
66
|
d = create_driver
|
49
67
|
|
@@ -56,10 +74,19 @@ class FileOutputTest < Test::Unit::TestCase
|
|
56
74
|
expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
|
57
75
|
assert_equal expect_path, path
|
58
76
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
77
|
+
check_gzipped_result(path, %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_write_with_format_json
|
81
|
+
d = create_driver [CONFIG, 'format json', 'include_time_key true', 'time_as_epoch'].join("\n")
|
82
|
+
|
83
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
84
|
+
d.emit({"a"=>1}, time)
|
85
|
+
d.emit({"a"=>2}, time)
|
86
|
+
|
87
|
+
# FileOutput#write returns path
|
88
|
+
path = d.run
|
89
|
+
check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time})}\n] + %[#{Yajl.dump({"a" => 2, 'time' => time})}\n])
|
63
90
|
end
|
64
91
|
|
65
92
|
def test_write_path_increment
|
@@ -68,16 +95,43 @@ class FileOutputTest < Test::Unit::TestCase
|
|
68
95
|
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
69
96
|
d.emit({"a"=>1}, time)
|
70
97
|
d.emit({"a"=>2}, time)
|
98
|
+
formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
71
99
|
|
72
100
|
# FileOutput#write returns path
|
73
101
|
path = d.run
|
74
102
|
assert_equal "#{TMP_DIR}/out_file_test._0.log.gz", path
|
75
|
-
|
103
|
+
check_gzipped_result(path, formatted_lines)
|
76
104
|
path = d.run
|
77
105
|
assert_equal "#{TMP_DIR}/out_file_test._1.log.gz", path
|
78
|
-
|
106
|
+
check_gzipped_result(path, formatted_lines)
|
79
107
|
path = d.run
|
80
108
|
assert_equal "#{TMP_DIR}/out_file_test._2.log.gz", path
|
109
|
+
check_gzipped_result(path, formatted_lines)
|
110
|
+
end
|
111
|
+
|
112
|
+
def test_write_with_append
|
113
|
+
d = create_driver %[
|
114
|
+
path #{TMP_DIR}/out_file_test
|
115
|
+
compress gz
|
116
|
+
utc
|
117
|
+
append true
|
118
|
+
]
|
119
|
+
|
120
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
121
|
+
d.emit({"a"=>1}, time)
|
122
|
+
d.emit({"a"=>2}, time)
|
123
|
+
formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
124
|
+
|
125
|
+
# FileOutput#write returns path
|
126
|
+
path = d.run
|
127
|
+
assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
|
128
|
+
check_gzipped_result(path, formatted_lines)
|
129
|
+
path = d.run
|
130
|
+
assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
|
131
|
+
check_gzipped_result(path, formatted_lines * 2)
|
132
|
+
path = d.run
|
133
|
+
assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
|
134
|
+
check_gzipped_result(path, formatted_lines * 3)
|
81
135
|
end
|
82
136
|
|
83
137
|
def test_write_with_symlink
|
@@ -0,0 +1,208 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/test'
|
3
|
+
require 'fluent/formatter'
|
4
|
+
|
5
|
+
module FormatterTest
|
6
|
+
include Fluent
|
7
|
+
|
8
|
+
def time2str(time, localtime = false, format = nil)
|
9
|
+
if format
|
10
|
+
if localtime
|
11
|
+
Time.at(time).strftime(format)
|
12
|
+
else
|
13
|
+
Time.at(time).utc.strftime(format)
|
14
|
+
end
|
15
|
+
else
|
16
|
+
if localtime
|
17
|
+
Time.at(time).iso8601
|
18
|
+
else
|
19
|
+
Time.at(time).utc.iso8601
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def tag
|
25
|
+
'tag'
|
26
|
+
end
|
27
|
+
|
28
|
+
def record
|
29
|
+
{'message' => 'awesome'}
|
30
|
+
end
|
31
|
+
|
32
|
+
class OutFileFormatterTest < ::Test::Unit::TestCase
|
33
|
+
include FormatterTest
|
34
|
+
|
35
|
+
def setup
|
36
|
+
@formatter = TextFormatter::TEMPLATE_REGISTRY.lookup('out_file').call
|
37
|
+
@time = Engine.now
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_format
|
41
|
+
@formatter.configure({})
|
42
|
+
formatted = @formatter.format(tag, @time, record)
|
43
|
+
|
44
|
+
assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_format_without_time
|
48
|
+
@formatter.configure('output_time' => 'false')
|
49
|
+
formatted = @formatter.format(tag, @time, record)
|
50
|
+
|
51
|
+
assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
|
52
|
+
end
|
53
|
+
|
54
|
+
def test_format_without_tag
|
55
|
+
@formatter.configure('output_tag' => 'false')
|
56
|
+
formatted = @formatter.format(tag, @time, record)
|
57
|
+
|
58
|
+
assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_format_without_time_and_tag
|
62
|
+
@formatter.configure('output_tag' => 'false', 'output_time' => 'false')
|
63
|
+
formatted = @formatter.format('tag', @time, record)
|
64
|
+
|
65
|
+
assert_equal("#{Yajl.dump(record)}\n", formatted)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
class JsonFormatterTest < ::Test::Unit::TestCase
|
70
|
+
include FormatterTest
|
71
|
+
|
72
|
+
def setup
|
73
|
+
@formatter = TextFormatter::JSONFormatter.new
|
74
|
+
@time = Engine.now
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_format
|
78
|
+
@formatter.configure({})
|
79
|
+
formatted = @formatter.format(tag, @time, record)
|
80
|
+
|
81
|
+
assert_equal("#{Yajl.dump(record)}\n", formatted)
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_format_with_include_tag
|
85
|
+
@formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo')
|
86
|
+
formatted = @formatter.format(tag, @time, record.dup)
|
87
|
+
|
88
|
+
r = record
|
89
|
+
r['foo'] = tag
|
90
|
+
assert_equal("#{Yajl.dump(r)}\n", formatted)
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_format_with_include_time
|
94
|
+
@formatter.configure('include_time_key' => 'true', 'localtime' => '')
|
95
|
+
formatted = @formatter.format(tag, @time, record.dup)
|
96
|
+
|
97
|
+
r = record
|
98
|
+
r['time'] = time2str(@time, true)
|
99
|
+
assert_equal("#{Yajl.dump(r)}\n", formatted)
|
100
|
+
end
|
101
|
+
|
102
|
+
def test_format_with_include_time_as_number
|
103
|
+
@formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch')
|
104
|
+
formatted = @formatter.format(tag, @time, record.dup)
|
105
|
+
|
106
|
+
r = record
|
107
|
+
r['epoch'] = @time
|
108
|
+
assert_equal("#{Yajl.dump(r)}\n", formatted)
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
class LabeledTSVFormatterTest < ::Test::Unit::TestCase
|
113
|
+
include FormatterTest
|
114
|
+
|
115
|
+
def setup
|
116
|
+
@formatter = TextFormatter::LabeledTSVFormatter.new
|
117
|
+
@time = Engine.now
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_config_params
|
121
|
+
assert_equal "\t", @formatter.delimiter
|
122
|
+
assert_equal ":", @formatter.label_delimiter
|
123
|
+
|
124
|
+
@formatter.configure(
|
125
|
+
'delimiter' => ',',
|
126
|
+
'label_delimiter' => '=',
|
127
|
+
)
|
128
|
+
|
129
|
+
assert_equal ",", @formatter.delimiter
|
130
|
+
assert_equal "=", @formatter.label_delimiter
|
131
|
+
end
|
132
|
+
|
133
|
+
def test_format
|
134
|
+
@formatter.configure({})
|
135
|
+
formatted = @formatter.format(tag, @time, record)
|
136
|
+
|
137
|
+
assert_equal("message:awesome\n", formatted)
|
138
|
+
end
|
139
|
+
|
140
|
+
def test_format_with_tag
|
141
|
+
@formatter.configure('include_tag_key' => 'true')
|
142
|
+
formatted = @formatter.format(tag, @time, record)
|
143
|
+
|
144
|
+
assert_equal("message:awesome\ttag:tag\n", formatted)
|
145
|
+
end
|
146
|
+
|
147
|
+
def test_format_with_time
|
148
|
+
@formatter.configure('include_time_key' => 'true', 'time_format' => '%Y')
|
149
|
+
formatted = @formatter.format(tag, @time, record)
|
150
|
+
|
151
|
+
assert_equal("message:awesome\ttime:#{Time.now.year}\n", formatted)
|
152
|
+
end
|
153
|
+
|
154
|
+
def test_format_with_customized_delimiters
|
155
|
+
@formatter.configure(
|
156
|
+
'include_tag_key' => 'true',
|
157
|
+
'delimiter' => ',',
|
158
|
+
'label_delimiter' => '=',
|
159
|
+
)
|
160
|
+
formatted = @formatter.format(tag, @time, record)
|
161
|
+
|
162
|
+
assert_equal("message=awesome,tag=tag\n", formatted)
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
class SingleValueFormatterTest < ::Test::Unit::TestCase
|
167
|
+
include FormatterTest
|
168
|
+
|
169
|
+
def test_config_params
|
170
|
+
formatter = TextFormatter::SingleValueFormatter.new
|
171
|
+
assert_equal "message", formatter.message_key
|
172
|
+
|
173
|
+
formatter.configure('message_key' => 'foobar')
|
174
|
+
assert_equal "foobar", formatter.message_key
|
175
|
+
end
|
176
|
+
|
177
|
+
def test_format
|
178
|
+
formatter = TextFormatter::TEMPLATE_REGISTRY.lookup('single_value').call
|
179
|
+
formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
|
180
|
+
assert_equal('awesome', formatted)
|
181
|
+
end
|
182
|
+
|
183
|
+
def test_format_with_message_key
|
184
|
+
formatter = TextFormatter::SingleValueFormatter.new
|
185
|
+
formatter.configure('message_key' => 'foobar')
|
186
|
+
formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})
|
187
|
+
|
188
|
+
assert_equal('foo', formatted)
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
class FormatterLookupTest < ::Test::Unit::TestCase
|
193
|
+
include FormatterTest
|
194
|
+
|
195
|
+
def test_unknown_format
|
196
|
+
assert_raise ConfigError do
|
197
|
+
TextFormatter::TEMPLATE_REGISTRY.lookup('unknown')
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
def test_find_formatter
|
202
|
+
$LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), 'scripts'))
|
203
|
+
assert_nothing_raised ConfigError do
|
204
|
+
TextFormatter::TEMPLATE_REGISTRY.lookup('known')
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.10.
|
4
|
+
version: 0.10.49
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-05
|
11
|
+
date: 2014-06-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|
@@ -287,6 +287,7 @@ files:
|
|
287
287
|
- lib/fluent/engine.rb
|
288
288
|
- lib/fluent/env.rb
|
289
289
|
- lib/fluent/event.rb
|
290
|
+
- lib/fluent/formatter.rb
|
290
291
|
- lib/fluent/input.rb
|
291
292
|
- lib/fluent/load.rb
|
292
293
|
- lib/fluent/log.rb
|
@@ -363,9 +364,11 @@ files:
|
|
363
364
|
- test/plugin/test_out_stdout.rb
|
364
365
|
- test/plugin/test_out_stream.rb
|
365
366
|
- test/scripts/exec_script.rb
|
367
|
+
- test/scripts/fluent/plugin/formatter_known.rb
|
366
368
|
- test/scripts/fluent/plugin/parser_known.rb
|
367
369
|
- test/test_config.rb
|
368
370
|
- test/test_configdsl.rb
|
371
|
+
- test/test_formatter.rb
|
369
372
|
- test/test_match.rb
|
370
373
|
- test/test_mixin.rb
|
371
374
|
- test/test_output.rb
|
@@ -427,9 +430,11 @@ test_files:
|
|
427
430
|
- test/plugin/test_out_stdout.rb
|
428
431
|
- test/plugin/test_out_stream.rb
|
429
432
|
- test/scripts/exec_script.rb
|
433
|
+
- test/scripts/fluent/plugin/formatter_known.rb
|
430
434
|
- test/scripts/fluent/plugin/parser_known.rb
|
431
435
|
- test/test_config.rb
|
432
436
|
- test/test_configdsl.rb
|
437
|
+
- test/test_formatter.rb
|
433
438
|
- test/test_match.rb
|
434
439
|
- test/test_mixin.rb
|
435
440
|
- test/test_output.rb
|