fluent-plugin-file-alternative 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +25 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/README.md +71 -0
- data/Rakefile +11 -0
- data/fluent-plugin-file-alternative.gemspec +20 -0
- data/lib/fluent/plugin/out_file_alternative.rb +347 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_file_alternative.rb +120 -0
- data/test/plugin/test_out_file_original.rb +81 -0
- metadata +80 -0
data/.gitignore
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.bundle
|
4
|
+
.config
|
5
|
+
.yardoc
|
6
|
+
Gemfile.lock
|
7
|
+
InstalledFiles
|
8
|
+
_yardoc
|
9
|
+
coverage
|
10
|
+
doc/
|
11
|
+
lib/bundler/man
|
12
|
+
pkg
|
13
|
+
rdoc
|
14
|
+
spec/reports
|
15
|
+
test/tmp
|
16
|
+
test/version_tmp
|
17
|
+
tmp
|
18
|
+
|
19
|
+
# For TextMate, emacs, vim
|
20
|
+
*.tmproj
|
21
|
+
tmtags
|
22
|
+
*~
|
23
|
+
\#*
|
24
|
+
.\#*
|
25
|
+
*.swp
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2012- TAGOMORI Satoshi
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
# fluent-plugin-file-alternative
|
2
|
+
|
3
|
+
File output plugin alternative implementation, **is 100% compatible with fluentd built-in 'out_file'**, and added many options to format output as you want.
|
4
|
+
|
5
|
+
FileAlternativeOutput slices data by time (for specified units), and store these data as plain text on hdfs. You can specify to:
|
6
|
+
|
7
|
+
* format whole data as serialized JSON, single attribute or separated multi attributes
|
8
|
+
* include time as line header, or not
|
9
|
+
* include tag as line header, or not
|
10
|
+
* change field separator (default: TAB)
|
11
|
+
* add new line as termination, or not
|
12
|
+
|
13
|
+
And you can specify output file path as:
|
14
|
+
|
15
|
+
* Standard out_file way
|
16
|
+
* configure 'path /path/to/dir/access'
|
17
|
+
* and 'time\_slice\_format %Y%m%d'
|
18
|
+
* got '/path/to/dir/access.20120316.log'
|
19
|
+
* Alternative style
|
20
|
+
* configure 'path /path/to/dir/access.%Y%m%d.log' only
|
21
|
+
* got '/path/to/dir/access.20120316.log'
|
22
|
+
|
23
|
+
And, gzip compression is also supported.
|
24
|
+
|
25
|
+
## Configuration
|
26
|
+
|
27
|
+
### FileAlternativeOutput
|
28
|
+
|
29
|
+
Standard out_file way (hourly log, compression, time-tag-json):
|
30
|
+
|
31
|
+
<match out.**>
|
32
|
+
type file\_alternative
|
33
|
+
path /var/log/service/access.*.log
|
34
|
+
time\_slice\_output %Y%m%d_%H
|
35
|
+
compress gzip
|
36
|
+
</match>
|
37
|
+
|
38
|
+
By this configuration, in gzip compressed file '/var/log/service/access.20120316_23.log.gz', you get:
|
39
|
+
|
40
|
+
2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
|
41
|
+
2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
|
42
|
+
2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
|
43
|
+
2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
|
44
|
+
|
45
|
+
If you don't want fluentd-time and tag in written file, and messages with single attribute (as raw full apache log with newline):
|
46
|
+
|
47
|
+
<match out.**>
|
48
|
+
type file\_alternative
|
49
|
+
path /var/log/service/access.%Y%m%d_%H.log
|
50
|
+
compress gzip
|
51
|
+
output_include_time false
|
52
|
+
output_include_tag false
|
53
|
+
output_data_type attr:message
|
54
|
+
add_newline false
|
55
|
+
</match>
|
56
|
+
|
57
|
+
Then, you will get:
|
58
|
+
|
59
|
+
192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
|
60
|
+
192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
|
61
|
+
192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
|
62
|
+
|
63
|
+
## TODO
|
64
|
+
|
65
|
+
* consider what to do next
|
66
|
+
* patches welcome!
|
67
|
+
|
68
|
+
## Copyright
|
69
|
+
|
70
|
+
Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
|
71
|
+
License:: Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-file-alternative"
|
5
|
+
gem.version = "0.1.0"
|
6
|
+
|
7
|
+
gem.authors = ["TAGOMORI Satoshi"]
|
8
|
+
gem.email = ["tagomoris@gmail.com"]
|
9
|
+
gem.description = %q{alternative implementation of out_file, with various configurations}
|
10
|
+
gem.summary = %q{alternative implementation of out_file}
|
11
|
+
gem.homepage = "https://github.com/tagomoris/fluent-plugin-file-alternative"
|
12
|
+
|
13
|
+
gem.files = `git ls-files`.split($\)
|
14
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
15
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
16
|
+
gem.require_paths = ["lib"]
|
17
|
+
|
18
|
+
gem.add_development_dependency "fluentd"
|
19
|
+
gem.add_runtime_dependency "fluentd"
|
20
|
+
end
|
@@ -0,0 +1,347 @@
|
|
1
|
+
module FluentExt; end
|
2
|
+
module FluentExt::PlainTextFormatterMixin
|
3
|
+
# config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
|
4
|
+
|
5
|
+
attr_accessor :output_include_time, :output_include_tag, :output_data_type
|
6
|
+
attr_accessor :add_newline, :field_separator
|
7
|
+
attr_accessor :remove_prefix, :default_tag
|
8
|
+
|
9
|
+
attr_accessor :f_separator
|
10
|
+
|
11
|
+
def configure(conf)
|
12
|
+
super
|
13
|
+
|
14
|
+
@output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
|
15
|
+
@output_include_time = true if @output_include_time.nil?
|
16
|
+
|
17
|
+
@output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
|
18
|
+
@output_include_tag = true if @output_include_tag.nil?
|
19
|
+
|
20
|
+
@output_data_type = conf['output_data_type']
|
21
|
+
@output_data_type = 'json' if @output_data_type.nil?
|
22
|
+
|
23
|
+
@f_separator = case conf['field_separator']
|
24
|
+
when 'SPACE' then ' '
|
25
|
+
when 'COMMA' then ','
|
26
|
+
else "\t"
|
27
|
+
end
|
28
|
+
@add_newline = Fluent::Config.bool_value(conf['add_newline'])
|
29
|
+
if @add_newline.nil?
|
30
|
+
@add_newline = true
|
31
|
+
end
|
32
|
+
|
33
|
+
@remove_prefix = conf['remove_prefix']
|
34
|
+
if @remove_prefix
|
35
|
+
@removed_prefix_string = @remove_prefix + '.'
|
36
|
+
@removed_length = @removed_prefix_string.length
|
37
|
+
end
|
38
|
+
if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
|
39
|
+
@default_tag = conf['default_tag']
|
40
|
+
if @default_tag.nil? or @default_tag.length < 1
|
41
|
+
raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# default timezone: utc
|
46
|
+
if conf['localtime'].nil? and conf['utc'].nil?
|
47
|
+
@utc = true
|
48
|
+
@localtime = false
|
49
|
+
elsif not @localtime and not @utc
|
50
|
+
@utc = true
|
51
|
+
@localtime = false
|
52
|
+
end
|
53
|
+
# mix-in default time formatter (or you can overwrite @timef on your own configure)
|
54
|
+
@timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
|
55
|
+
|
56
|
+
@custom_attributes = []
|
57
|
+
if @output_data_type == 'json'
|
58
|
+
self.instance_eval {
|
59
|
+
def stringify_record(record)
|
60
|
+
record.to_json
|
61
|
+
end
|
62
|
+
}
|
63
|
+
elsif @output_data_type =~ /^attr:(.*)$/
|
64
|
+
@custom_attributes = $1.split(',')
|
65
|
+
if @custom_attributes.size > 1
|
66
|
+
self.instance_eval {
|
67
|
+
def stringify_record(record)
|
68
|
+
@custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@f_separator)
|
69
|
+
end
|
70
|
+
}
|
71
|
+
elsif @custom_attributes.size == 1
|
72
|
+
self.instance_eval {
|
73
|
+
def stringify_record(record)
|
74
|
+
(record[@custom_attributes[0]] || 'NULL').to_s
|
75
|
+
end
|
76
|
+
}
|
77
|
+
else
|
78
|
+
raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
|
79
|
+
end
|
80
|
+
else
|
81
|
+
raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
|
82
|
+
end
|
83
|
+
|
84
|
+
if @output_include_time and @output_include_tag
|
85
|
+
if @add_newline and @remove_prefix
|
86
|
+
self.instance_eval {
|
87
|
+
def format(tag,time,record)
|
88
|
+
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
89
|
+
tag == @remove_prefix
|
90
|
+
tag = tag[@removed_length..-1] || @default_tag
|
91
|
+
end
|
92
|
+
@timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
|
93
|
+
end
|
94
|
+
}
|
95
|
+
elsif @add_newline
|
96
|
+
self.instance_eval {
|
97
|
+
def format(tag,time,record)
|
98
|
+
@timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
|
99
|
+
end
|
100
|
+
}
|
101
|
+
elsif @remove_prefix
|
102
|
+
self.instance_eval {
|
103
|
+
def format(tag,time,record)
|
104
|
+
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
105
|
+
tag == @remove_prefix
|
106
|
+
tag = tag[@removed_length..-1] || @default_tag
|
107
|
+
end
|
108
|
+
@timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
|
109
|
+
end
|
110
|
+
}
|
111
|
+
else
|
112
|
+
self.instance_eval {
|
113
|
+
def format(tag,time,record)
|
114
|
+
@timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
|
115
|
+
end
|
116
|
+
}
|
117
|
+
end
|
118
|
+
elsif @output_include_time
|
119
|
+
if @add_newline
|
120
|
+
self.instance_eval {
|
121
|
+
def format(tag,time,record);
|
122
|
+
@timef.format(time) + @f_separator + stringify_record(record) + "\n"
|
123
|
+
end
|
124
|
+
}
|
125
|
+
else
|
126
|
+
self.instance_eval {
|
127
|
+
def format(tag,time,record);
|
128
|
+
@timef.format(time) + @f_separator + stringify_record(record)
|
129
|
+
end
|
130
|
+
}
|
131
|
+
end
|
132
|
+
elsif @output_include_tag
|
133
|
+
if @add_newline and @remove_prefix
|
134
|
+
self.instance_eval {
|
135
|
+
def format(tag,time,record)
|
136
|
+
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
137
|
+
tag == @remove_prefix
|
138
|
+
tag = tag[@removed_length..-1] || @default_tag
|
139
|
+
end
|
140
|
+
tag + @f_separator + stringify_record(record) + "\n"
|
141
|
+
end
|
142
|
+
}
|
143
|
+
elsif @add_newline
|
144
|
+
self.instance_eval {
|
145
|
+
def format(tag,time,record)
|
146
|
+
tag + @f_separator + stringify_record(record) + "\n"
|
147
|
+
end
|
148
|
+
}
|
149
|
+
elsif @remove_prefix
|
150
|
+
self.instance_eval {
|
151
|
+
def format(tag,time,record)
|
152
|
+
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
153
|
+
tag == @remove_prefix
|
154
|
+
tag = tag[@removed_length..-1] || @default_tag
|
155
|
+
end
|
156
|
+
tag + @f_separator + stringify_record(record)
|
157
|
+
end
|
158
|
+
}
|
159
|
+
else
|
160
|
+
self.instance_eval {
|
161
|
+
def format(tag,time,record)
|
162
|
+
tag + @f_separator + stringify_record(record)
|
163
|
+
end
|
164
|
+
}
|
165
|
+
end
|
166
|
+
else # without time, tag
|
167
|
+
if @add_newline
|
168
|
+
self.instance_eval {
|
169
|
+
def format(tag,time,record);
|
170
|
+
stringify_record(record) + "\n"
|
171
|
+
end
|
172
|
+
}
|
173
|
+
else
|
174
|
+
self.instance_eval {
|
175
|
+
def format(tag,time,record);
|
176
|
+
stringify_record(record)
|
177
|
+
end
|
178
|
+
}
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def stringify_record(record)
|
184
|
+
record.to_json
|
185
|
+
end
|
186
|
+
|
187
|
+
def format(tag, time, record)
|
188
|
+
if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
|
189
|
+
tag = tag[@removed_length..-1] || @default_tag
|
190
|
+
end
|
191
|
+
time_str = if @output_include_time
|
192
|
+
@timef.format(time) + @f_separator
|
193
|
+
else
|
194
|
+
''
|
195
|
+
end
|
196
|
+
tag_str = if @output_include_tag
|
197
|
+
tag + @f_separator
|
198
|
+
else
|
199
|
+
''
|
200
|
+
end
|
201
|
+
time_str + tag_str + stringify_record(record) + "\n"
|
202
|
+
end
|
203
|
+
|
204
|
+
end
|
205
|
+
|
206
|
+
class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
|
207
|
+
Fluent::Plugin.register_output('file_alternative', self)
|
208
|
+
|
209
|
+
SUPPORTED_COMPRESS = {
|
210
|
+
:gz => :gz,
|
211
|
+
:gzip => :gz,
|
212
|
+
}
|
213
|
+
|
214
|
+
config_set_default :time_slice_format, '%Y%m%d' # %Y%m%d%H
|
215
|
+
|
216
|
+
config_param :path, :string # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S
|
217
|
+
|
218
|
+
config_param :compress, :default => nil do |val|
|
219
|
+
c = SUPPORTED_COMPRESS[val.to_sym]
|
220
|
+
unless c
|
221
|
+
raise ConfigError, "Unsupported compression algorithm '#{compress}'"
|
222
|
+
end
|
223
|
+
c
|
224
|
+
end
|
225
|
+
|
226
|
+
include FluentExt::PlainTextFormatterMixin
|
227
|
+
|
228
|
+
config_set_default :output_include_time, true
|
229
|
+
config_set_default :output_include_tag, true
|
230
|
+
config_set_default :output_data_type, 'json'
|
231
|
+
config_set_default :field_separator, "\t"
|
232
|
+
config_set_default :add_newline, true
|
233
|
+
config_set_default :remove_prefix, nil
|
234
|
+
|
235
|
+
def initialize
|
236
|
+
super
|
237
|
+
require 'time'
|
238
|
+
require 'zlib'
|
239
|
+
end
|
240
|
+
|
241
|
+
def configure(conf)
|
242
|
+
if conf['path']
|
243
|
+
if conf['path'].index('%S')
|
244
|
+
conf['time_slice_format'] = '%Y%m%d%H%M%S'
|
245
|
+
elsif conf['path'].index('%M')
|
246
|
+
conf['time_slice_format'] = '%Y%m%d%H%M'
|
247
|
+
elsif conf['path'].index('%H')
|
248
|
+
conf['time_slice_format'] = '%Y%m%d%H'
|
249
|
+
end
|
250
|
+
end
|
251
|
+
if pos = (conf['path'] || '').index('*')
|
252
|
+
@path_prefix = conf['path'][0,pos]
|
253
|
+
@path_suffix = conf['path'][pos+1..-1]
|
254
|
+
conf['buffer_path'] ||= "#{conf['path']}"
|
255
|
+
elsif (conf['path'] || '%Y') !~ /%Y|%m|%d|%H|%M|%S/
|
256
|
+
@path_prefix = conf['path'] + "."
|
257
|
+
@path_suffix = ".log"
|
258
|
+
conf['buffer_path'] ||= "#{conf['path']}.*"
|
259
|
+
elsif (conf['path'] || '') =~ /%Y|%m|%d|%H|%M|%S/
|
260
|
+
conf['buffer_path'] ||= conf['path'].gsub('%Y','yyyy').gsub('%m','mm').gsub('%d','dd').gsub('%H','HH').gsub('%M','MM').gsub('%S','SS')
|
261
|
+
end
|
262
|
+
|
263
|
+
super
|
264
|
+
|
265
|
+
unless @path.index('/') == 0
|
266
|
+
raise Fluent::ConfigError, "Path on filesystem MUST starts with '/', but '#{@path}'"
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
def start
|
271
|
+
super
|
272
|
+
# init
|
273
|
+
end
|
274
|
+
|
275
|
+
def shutdown
|
276
|
+
super
|
277
|
+
# destroy
|
278
|
+
end
|
279
|
+
|
280
|
+
def record_to_string(record)
|
281
|
+
record.to_json
|
282
|
+
end
|
283
|
+
|
284
|
+
def format(tag, time, record)
|
285
|
+
time_str = @timef.format(time)
|
286
|
+
time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
|
287
|
+
end
|
288
|
+
|
289
|
+
def path_format(chunk_key)
|
290
|
+
suffix = case @compress
|
291
|
+
when :gz
|
292
|
+
'.gz'
|
293
|
+
else
|
294
|
+
''
|
295
|
+
end
|
296
|
+
if @path_prefix and @path_suffix
|
297
|
+
if @compress
|
298
|
+
i = 0
|
299
|
+
begin
|
300
|
+
path = "#{@path_prefix}#{chunk_key}_#{i}#{@path_suffix}#{suffix}"
|
301
|
+
i += 1
|
302
|
+
end while File.exist?(path)
|
303
|
+
path
|
304
|
+
else
|
305
|
+
"#{@path_prefix}#{chunk_key}_#{@path_suffix}#{suffix}"
|
306
|
+
end
|
307
|
+
else
|
308
|
+
if @compress
|
309
|
+
path_base = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
|
310
|
+
path = path_base + suffix
|
311
|
+
if File.exist?(path)
|
312
|
+
i = 0
|
313
|
+
begin
|
314
|
+
path = "#{path_base}.#{i}#{suffix}"
|
315
|
+
i += 1
|
316
|
+
end while File.exist?(path)
|
317
|
+
end
|
318
|
+
path
|
319
|
+
else
|
320
|
+
Time.strptime(chunk_key, @time_slice_format).strftime(@path)
|
321
|
+
end
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
def write(chunk)
|
326
|
+
path = path_format(chunk.key)
|
327
|
+
|
328
|
+
begin
|
329
|
+
FileUtils.mkdir_p File.dirname(path)
|
330
|
+
|
331
|
+
case @compress
|
332
|
+
when :gz
|
333
|
+
Zlib::GzipWriter.open(path) {|f|
|
334
|
+
chunk.write_to(f)
|
335
|
+
}
|
336
|
+
else
|
337
|
+
File.open(path, "a") {|f|
|
338
|
+
chunk.write_to(f)
|
339
|
+
}
|
340
|
+
end
|
341
|
+
rescue
|
342
|
+
$log.error "failed to write data: path #{path}"
|
343
|
+
raise
|
344
|
+
end
|
345
|
+
path
|
346
|
+
end
|
347
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_file_alternative'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class FileAlternativeOutputTest < Test::Unit::TestCase
|
4
|
+
TMP_DIR = File.dirname(__FILE__) + "/../tmp"
|
5
|
+
|
6
|
+
CONFIG = %[
|
7
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d
|
8
|
+
compress gz
|
9
|
+
]
|
10
|
+
|
11
|
+
def setup
|
12
|
+
Fluent::Test.setup
|
13
|
+
FileUtils.rm_rf(TMP_DIR)
|
14
|
+
FileUtils.mkdir_p(TMP_DIR)
|
15
|
+
end
|
16
|
+
|
17
|
+
def create_driver(conf = CONFIG, tag='test')
|
18
|
+
Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::FileAlternativeOutput, tag).configure(conf)
|
19
|
+
end
|
20
|
+
|
21
|
+
# config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
|
22
|
+
|
23
|
+
# path
|
24
|
+
# output_include_time , output_include_tag , output_data_type
|
25
|
+
# add_newline , field_separator
|
26
|
+
# remove_prefix , :default_tag
|
27
|
+
def test_configure
|
28
|
+
# many many many tests should be written, for PlainTextFormatterMixin ...
|
29
|
+
|
30
|
+
d = create_driver %[
|
31
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
32
|
+
]
|
33
|
+
assert_equal '%Y%m%d%H%M%S', d.instance.time_slice_format
|
34
|
+
assert_nil d.instance.compress
|
35
|
+
assert_equal true, d.instance.output_include_time
|
36
|
+
assert_equal true, d.instance.output_include_tag
|
37
|
+
assert_equal 'json', d.instance.output_data_type
|
38
|
+
assert_equal true, d.instance.add_newline
|
39
|
+
assert_equal "\t", d.instance.field_separator
|
40
|
+
assert_nil d.instance.remove_prefix
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_format
|
44
|
+
d1 = create_driver
|
45
|
+
|
46
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
47
|
+
d1.emit({"a"=>1}, time)
|
48
|
+
d1.emit({"a"=>2}, time)
|
49
|
+
d1.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
|
50
|
+
d1.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
51
|
+
d1.run
|
52
|
+
|
53
|
+
d2 = create_driver %[
|
54
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
55
|
+
output_include_time false
|
56
|
+
output_include_tag false
|
57
|
+
output_data_type attr:message
|
58
|
+
]
|
59
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
60
|
+
d2.emit({"message" => "abc-xyz-123", "other" => "zzz"}, time)
|
61
|
+
d2.emit({"message" => "123-456-789", "other" => "ppp"}, time)
|
62
|
+
d2.expect_format %[abc-xyz-123\n]
|
63
|
+
d2.expect_format %[123-456-789\n]
|
64
|
+
d2.run
|
65
|
+
|
66
|
+
d3 = create_driver %[
|
67
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
68
|
+
output_include_time false
|
69
|
+
output_include_tag false
|
70
|
+
output_data_type attr:server,level,log
|
71
|
+
field_separator COMMA
|
72
|
+
add_newline false
|
73
|
+
]
|
74
|
+
assert_equal ',', d3.instance.f_separator
|
75
|
+
|
76
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
77
|
+
d3.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
|
78
|
+
d3.emit({"server" => "app01", "level" => "info", "log" => "Send response\n"}, time)
|
79
|
+
d3.expect_format %[www01,warn,Exception\n]
|
80
|
+
d3.expect_format %[app01,info,Send response\n]
|
81
|
+
d3.run
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_write
|
85
|
+
d2 = create_driver %[
|
86
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
87
|
+
output_include_time false
|
88
|
+
output_include_tag false
|
89
|
+
output_data_type attr:message
|
90
|
+
utc
|
91
|
+
]
|
92
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
93
|
+
d2.emit({"message" => "abc-xyz-123", "other" => "zzz"}, time)
|
94
|
+
d2.emit({"message" => "123-456-789", "other" => "ppp"}, time)
|
95
|
+
d2.expect_format %[abc-xyz-123\n]
|
96
|
+
d2.expect_format %[123-456-789\n]
|
97
|
+
path = d2.run
|
98
|
+
assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15", path[0]
|
99
|
+
|
100
|
+
d3 = create_driver %[
|
101
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
102
|
+
compress gzip
|
103
|
+
output_include_time false
|
104
|
+
output_include_tag false
|
105
|
+
output_data_type attr:server,level,log
|
106
|
+
field_separator COMMA
|
107
|
+
add_newline false
|
108
|
+
utc
|
109
|
+
]
|
110
|
+
assert_equal ',', d3.instance.f_separator
|
111
|
+
|
112
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
113
|
+
d3.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
|
114
|
+
d3.emit({"server" => "app01", "level" => "info", "log" => "Send response\n"}, time)
|
115
|
+
d3.expect_format %[www01,warn,Exception\n]
|
116
|
+
d3.expect_format %[app01,info,Send response\n]
|
117
|
+
path = d3.run
|
118
|
+
assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15.gz", path[0]
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'time'
|
4
|
+
|
5
|
+
class FileOutputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
FileUtils.rm_rf(TMP_DIR)
|
9
|
+
FileUtils.mkdir_p(TMP_DIR)
|
10
|
+
end
|
11
|
+
|
12
|
+
TMP_DIR = File.dirname(__FILE__) + "/../tmp"
|
13
|
+
|
14
|
+
CONFIG = %[
|
15
|
+
path #{TMP_DIR}/out_file_test
|
16
|
+
compress gz
|
17
|
+
utc
|
18
|
+
]
|
19
|
+
|
20
|
+
def create_driver(conf = CONFIG)
|
21
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_configure
|
25
|
+
d = create_driver %[
|
26
|
+
path test_path
|
27
|
+
compress gz
|
28
|
+
]
|
29
|
+
assert_equal 'test_path', d.instance.path
|
30
|
+
assert_equal :gz, d.instance.compress
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_format
|
34
|
+
d = create_driver
|
35
|
+
|
36
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
37
|
+
d.emit({"a"=>1}, time)
|
38
|
+
d.emit({"a"=>2}, time)
|
39
|
+
|
40
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
|
41
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
42
|
+
|
43
|
+
d.run
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_write
|
47
|
+
d = create_driver
|
48
|
+
|
49
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
50
|
+
d.emit({"a"=>1}, time)
|
51
|
+
d.emit({"a"=>2}, time)
|
52
|
+
|
53
|
+
# FileOutput#write returns path
|
54
|
+
path = d.run
|
55
|
+
expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
|
56
|
+
assert_equal expect_path, path
|
57
|
+
|
58
|
+
data = Zlib::GzipReader.open(expect_path) {|f| f.read }
|
59
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
60
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
61
|
+
data
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_write_path_increment
|
65
|
+
d = create_driver
|
66
|
+
|
67
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
68
|
+
d.emit({"a"=>1}, time)
|
69
|
+
d.emit({"a"=>2}, time)
|
70
|
+
|
71
|
+
# FileOutput#write returns path
|
72
|
+
path = d.run
|
73
|
+
assert_equal "#{TMP_DIR}/out_file_test._0.log.gz", path
|
74
|
+
|
75
|
+
path = d.run
|
76
|
+
assert_equal "#{TMP_DIR}/out_file_test._1.log.gz", path
|
77
|
+
|
78
|
+
path = d.run
|
79
|
+
assert_equal "#{TMP_DIR}/out_file_test._2.log.gz", path
|
80
|
+
end
|
81
|
+
end
|
metadata
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-file-alternative
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- TAGOMORI Satoshi
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-03-16 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: &2168521620 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *2168521620
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: fluentd
|
27
|
+
requirement: &2168521180 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *2168521180
|
36
|
+
description: alternative implementation of out_file, with various configurations
|
37
|
+
email:
|
38
|
+
- tagomoris@gmail.com
|
39
|
+
executables: []
|
40
|
+
extensions: []
|
41
|
+
extra_rdoc_files: []
|
42
|
+
files:
|
43
|
+
- .gitignore
|
44
|
+
- Gemfile
|
45
|
+
- LICENSE.txt
|
46
|
+
- README.md
|
47
|
+
- Rakefile
|
48
|
+
- fluent-plugin-file-alternative.gemspec
|
49
|
+
- lib/fluent/plugin/out_file_alternative.rb
|
50
|
+
- test/helper.rb
|
51
|
+
- test/plugin/test_out_file_alternative.rb
|
52
|
+
- test/plugin/test_out_file_original.rb
|
53
|
+
homepage: https://github.com/tagomoris/fluent-plugin-file-alternative
|
54
|
+
licenses: []
|
55
|
+
post_install_message:
|
56
|
+
rdoc_options: []
|
57
|
+
require_paths:
|
58
|
+
- lib
|
59
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
60
|
+
none: false
|
61
|
+
requirements:
|
62
|
+
- - ! '>='
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '0'
|
65
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ! '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
requirements: []
|
72
|
+
rubyforge_project:
|
73
|
+
rubygems_version: 1.8.6
|
74
|
+
signing_key:
|
75
|
+
specification_version: 3
|
76
|
+
summary: alternative implementation of out_file
|
77
|
+
test_files:
|
78
|
+
- test/helper.rb
|
79
|
+
- test/plugin/test_out_file_alternative.rb
|
80
|
+
- test/plugin/test_out_file_original.rb
|