fluent-plugin-file-alternative 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +25 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/README.md +71 -0
- data/Rakefile +11 -0
- data/fluent-plugin-file-alternative.gemspec +20 -0
- data/lib/fluent/plugin/out_file_alternative.rb +347 -0
- data/test/helper.rb +28 -0
- data/test/plugin/test_out_file_alternative.rb +120 -0
- data/test/plugin/test_out_file_original.rb +81 -0
- metadata +80 -0
data/.gitignore
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.bundle
|
4
|
+
.config
|
5
|
+
.yardoc
|
6
|
+
Gemfile.lock
|
7
|
+
InstalledFiles
|
8
|
+
_yardoc
|
9
|
+
coverage
|
10
|
+
doc/
|
11
|
+
lib/bundler/man
|
12
|
+
pkg
|
13
|
+
rdoc
|
14
|
+
spec/reports
|
15
|
+
test/tmp
|
16
|
+
test/version_tmp
|
17
|
+
tmp
|
18
|
+
|
19
|
+
# For TextMate, emacs, vim
|
20
|
+
*.tmproj
|
21
|
+
tmtags
|
22
|
+
*~
|
23
|
+
\#*
|
24
|
+
.\#*
|
25
|
+
*.swp
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2012- TAGOMORI Satoshi
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
# fluent-plugin-file-alternative
|
2
|
+
|
3
|
+
File output plugin alternative implementation, **is 100% compatible with fluentd built-in 'out_file'**, and added many options to format output as you want.
|
4
|
+
|
5
|
+
FileAlternativeOutput slices data by time (for specified units), and store these data as plain text on hdfs. You can specify to:
|
6
|
+
|
7
|
+
* format whole data as serialized JSON, single attribute or separated multi attributes
|
8
|
+
* include time as line header, or not
|
9
|
+
* include tag as line header, or not
|
10
|
+
* change field separator (default: TAB)
|
11
|
+
* add new line as termination, or not
|
12
|
+
|
13
|
+
And you can specify output file path as:
|
14
|
+
|
15
|
+
* Standard out_file way
|
16
|
+
* configure 'path /path/to/dir/access'
|
17
|
+
* and 'time\_slice\_format %Y%m%d'
|
18
|
+
* got '/path/to/dir/access.20120316.log'
|
19
|
+
* Alternative style
|
20
|
+
* configure 'path /path/to/dir/access.%Y%m%d.log' only
|
21
|
+
* got '/path/to/dir/access.20120316.log'
|
22
|
+
|
23
|
+
And, gzip compression is also supported.
|
24
|
+
|
25
|
+
## Configuration
|
26
|
+
|
27
|
+
### FileAlternativeOutput
|
28
|
+
|
29
|
+
Standard out_file way (hourly log, compression, time-tag-json):
|
30
|
+
|
31
|
+
<match out.**>
|
32
|
+
type file\_alternative
|
33
|
+
path /var/log/service/access.*.log
|
34
|
+
time\_slice\_output %Y%m%d_%H
|
35
|
+
compress gzip
|
36
|
+
</match>
|
37
|
+
|
38
|
+
By this configuration, in gzip compressed file '/var/log/service/access.20120316_23.log.gz', you get:
|
39
|
+
|
40
|
+
2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
|
41
|
+
2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
|
42
|
+
2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
|
43
|
+
2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
|
44
|
+
|
45
|
+
If you don't want fluentd-time and tag in written file, and messages with single attribute (as raw full apache log with newline):
|
46
|
+
|
47
|
+
<match out.**>
|
48
|
+
type file\_alternative
|
49
|
+
path /var/log/service/access.%Y%m%d_%H.log
|
50
|
+
compress gzip
|
51
|
+
output_include_time false
|
52
|
+
output_include_tag false
|
53
|
+
output_data_type attr:message
|
54
|
+
add_newline false
|
55
|
+
</match>
|
56
|
+
|
57
|
+
Then, you will get:
|
58
|
+
|
59
|
+
192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
|
60
|
+
192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
|
61
|
+
192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
|
62
|
+
|
63
|
+
## TODO
|
64
|
+
|
65
|
+
* consider what to do next
|
66
|
+
* patches welcome!
|
67
|
+
|
68
|
+
## Copyright
|
69
|
+
|
70
|
+
Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
|
71
|
+
License:: Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-file-alternative"
|
5
|
+
gem.version = "0.1.0"
|
6
|
+
|
7
|
+
gem.authors = ["TAGOMORI Satoshi"]
|
8
|
+
gem.email = ["tagomoris@gmail.com"]
|
9
|
+
gem.description = %q{alternative implementation of out_file, with various configurations}
|
10
|
+
gem.summary = %q{alternative implementation of out_file}
|
11
|
+
gem.homepage = "https://github.com/tagomoris/fluent-plugin-file-alternative"
|
12
|
+
|
13
|
+
gem.files = `git ls-files`.split($\)
|
14
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
15
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
16
|
+
gem.require_paths = ["lib"]
|
17
|
+
|
18
|
+
gem.add_development_dependency "fluentd"
|
19
|
+
gem.add_runtime_dependency "fluentd"
|
20
|
+
end
|
@@ -0,0 +1,347 @@
|
|
1
|
+
module FluentExt; end
|
2
|
+
module FluentExt::PlainTextFormatterMixin
|
3
|
+
# config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
|
4
|
+
|
5
|
+
attr_accessor :output_include_time, :output_include_tag, :output_data_type
|
6
|
+
attr_accessor :add_newline, :field_separator
|
7
|
+
attr_accessor :remove_prefix, :default_tag
|
8
|
+
|
9
|
+
attr_accessor :f_separator
|
10
|
+
|
11
|
+
def configure(conf)
|
12
|
+
super
|
13
|
+
|
14
|
+
@output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
|
15
|
+
@output_include_time = true if @output_include_time.nil?
|
16
|
+
|
17
|
+
@output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
|
18
|
+
@output_include_tag = true if @output_include_tag.nil?
|
19
|
+
|
20
|
+
@output_data_type = conf['output_data_type']
|
21
|
+
@output_data_type = 'json' if @output_data_type.nil?
|
22
|
+
|
23
|
+
@f_separator = case conf['field_separator']
|
24
|
+
when 'SPACE' then ' '
|
25
|
+
when 'COMMA' then ','
|
26
|
+
else "\t"
|
27
|
+
end
|
28
|
+
@add_newline = Fluent::Config.bool_value(conf['add_newline'])
|
29
|
+
if @add_newline.nil?
|
30
|
+
@add_newline = true
|
31
|
+
end
|
32
|
+
|
33
|
+
@remove_prefix = conf['remove_prefix']
|
34
|
+
if @remove_prefix
|
35
|
+
@removed_prefix_string = @remove_prefix + '.'
|
36
|
+
@removed_length = @removed_prefix_string.length
|
37
|
+
end
|
38
|
+
if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
|
39
|
+
@default_tag = conf['default_tag']
|
40
|
+
if @default_tag.nil? or @default_tag.length < 1
|
41
|
+
raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# default timezone: utc
|
46
|
+
if conf['localtime'].nil? and conf['utc'].nil?
|
47
|
+
@utc = true
|
48
|
+
@localtime = false
|
49
|
+
elsif not @localtime and not @utc
|
50
|
+
@utc = true
|
51
|
+
@localtime = false
|
52
|
+
end
|
53
|
+
# mix-in default time formatter (or you can overwrite @timef on your own configure)
|
54
|
+
@timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
|
55
|
+
|
56
|
+
@custom_attributes = []
|
57
|
+
if @output_data_type == 'json'
|
58
|
+
self.instance_eval {
|
59
|
+
def stringify_record(record)
|
60
|
+
record.to_json
|
61
|
+
end
|
62
|
+
}
|
63
|
+
elsif @output_data_type =~ /^attr:(.*)$/
|
64
|
+
@custom_attributes = $1.split(',')
|
65
|
+
if @custom_attributes.size > 1
|
66
|
+
self.instance_eval {
|
67
|
+
def stringify_record(record)
|
68
|
+
@custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@f_separator)
|
69
|
+
end
|
70
|
+
}
|
71
|
+
elsif @custom_attributes.size == 1
|
72
|
+
self.instance_eval {
|
73
|
+
def stringify_record(record)
|
74
|
+
(record[@custom_attributes[0]] || 'NULL').to_s
|
75
|
+
end
|
76
|
+
}
|
77
|
+
else
|
78
|
+
raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
|
79
|
+
end
|
80
|
+
else
|
81
|
+
raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
|
82
|
+
end
|
83
|
+
|
84
|
+
if @output_include_time and @output_include_tag
|
85
|
+
if @add_newline and @remove_prefix
|
86
|
+
self.instance_eval {
|
87
|
+
def format(tag,time,record)
|
88
|
+
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
89
|
+
tag == @remove_prefix
|
90
|
+
tag = tag[@removed_length..-1] || @default_tag
|
91
|
+
end
|
92
|
+
@timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
|
93
|
+
end
|
94
|
+
}
|
95
|
+
elsif @add_newline
|
96
|
+
self.instance_eval {
|
97
|
+
def format(tag,time,record)
|
98
|
+
@timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
|
99
|
+
end
|
100
|
+
}
|
101
|
+
elsif @remove_prefix
|
102
|
+
self.instance_eval {
|
103
|
+
def format(tag,time,record)
|
104
|
+
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
105
|
+
tag == @remove_prefix
|
106
|
+
tag = tag[@removed_length..-1] || @default_tag
|
107
|
+
end
|
108
|
+
@timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
|
109
|
+
end
|
110
|
+
}
|
111
|
+
else
|
112
|
+
self.instance_eval {
|
113
|
+
def format(tag,time,record)
|
114
|
+
@timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
|
115
|
+
end
|
116
|
+
}
|
117
|
+
end
|
118
|
+
elsif @output_include_time
|
119
|
+
if @add_newline
|
120
|
+
self.instance_eval {
|
121
|
+
def format(tag,time,record);
|
122
|
+
@timef.format(time) + @f_separator + stringify_record(record) + "\n"
|
123
|
+
end
|
124
|
+
}
|
125
|
+
else
|
126
|
+
self.instance_eval {
|
127
|
+
def format(tag,time,record);
|
128
|
+
@timef.format(time) + @f_separator + stringify_record(record)
|
129
|
+
end
|
130
|
+
}
|
131
|
+
end
|
132
|
+
elsif @output_include_tag
|
133
|
+
if @add_newline and @remove_prefix
|
134
|
+
self.instance_eval {
|
135
|
+
def format(tag,time,record)
|
136
|
+
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
137
|
+
tag == @remove_prefix
|
138
|
+
tag = tag[@removed_length..-1] || @default_tag
|
139
|
+
end
|
140
|
+
tag + @f_separator + stringify_record(record) + "\n"
|
141
|
+
end
|
142
|
+
}
|
143
|
+
elsif @add_newline
|
144
|
+
self.instance_eval {
|
145
|
+
def format(tag,time,record)
|
146
|
+
tag + @f_separator + stringify_record(record) + "\n"
|
147
|
+
end
|
148
|
+
}
|
149
|
+
elsif @remove_prefix
|
150
|
+
self.instance_eval {
|
151
|
+
def format(tag,time,record)
|
152
|
+
if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
|
153
|
+
tag == @remove_prefix
|
154
|
+
tag = tag[@removed_length..-1] || @default_tag
|
155
|
+
end
|
156
|
+
tag + @f_separator + stringify_record(record)
|
157
|
+
end
|
158
|
+
}
|
159
|
+
else
|
160
|
+
self.instance_eval {
|
161
|
+
def format(tag,time,record)
|
162
|
+
tag + @f_separator + stringify_record(record)
|
163
|
+
end
|
164
|
+
}
|
165
|
+
end
|
166
|
+
else # without time, tag
|
167
|
+
if @add_newline
|
168
|
+
self.instance_eval {
|
169
|
+
def format(tag,time,record);
|
170
|
+
stringify_record(record) + "\n"
|
171
|
+
end
|
172
|
+
}
|
173
|
+
else
|
174
|
+
self.instance_eval {
|
175
|
+
def format(tag,time,record);
|
176
|
+
stringify_record(record)
|
177
|
+
end
|
178
|
+
}
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def stringify_record(record)
|
184
|
+
record.to_json
|
185
|
+
end
|
186
|
+
|
187
|
+
def format(tag, time, record)
|
188
|
+
if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
|
189
|
+
tag = tag[@removed_length..-1] || @default_tag
|
190
|
+
end
|
191
|
+
time_str = if @output_include_time
|
192
|
+
@timef.format(time) + @f_separator
|
193
|
+
else
|
194
|
+
''
|
195
|
+
end
|
196
|
+
tag_str = if @output_include_tag
|
197
|
+
tag + @f_separator
|
198
|
+
else
|
199
|
+
''
|
200
|
+
end
|
201
|
+
time_str + tag_str + stringify_record(record) + "\n"
|
202
|
+
end
|
203
|
+
|
204
|
+
end
|
205
|
+
|
206
|
+
class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
|
207
|
+
Fluent::Plugin.register_output('file_alternative', self)
|
208
|
+
|
209
|
+
SUPPORTED_COMPRESS = {
|
210
|
+
:gz => :gz,
|
211
|
+
:gzip => :gz,
|
212
|
+
}
|
213
|
+
|
214
|
+
config_set_default :time_slice_format, '%Y%m%d' # %Y%m%d%H
|
215
|
+
|
216
|
+
config_param :path, :string # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S
|
217
|
+
|
218
|
+
config_param :compress, :default => nil do |val|
|
219
|
+
c = SUPPORTED_COMPRESS[val.to_sym]
|
220
|
+
unless c
|
221
|
+
raise ConfigError, "Unsupported compression algorithm '#{compress}'"
|
222
|
+
end
|
223
|
+
c
|
224
|
+
end
|
225
|
+
|
226
|
+
include FluentExt::PlainTextFormatterMixin
|
227
|
+
|
228
|
+
config_set_default :output_include_time, true
|
229
|
+
config_set_default :output_include_tag, true
|
230
|
+
config_set_default :output_data_type, 'json'
|
231
|
+
config_set_default :field_separator, "\t"
|
232
|
+
config_set_default :add_newline, true
|
233
|
+
config_set_default :remove_prefix, nil
|
234
|
+
|
235
|
+
def initialize
|
236
|
+
super
|
237
|
+
require 'time'
|
238
|
+
require 'zlib'
|
239
|
+
end
|
240
|
+
|
241
|
+
def configure(conf)
|
242
|
+
if conf['path']
|
243
|
+
if conf['path'].index('%S')
|
244
|
+
conf['time_slice_format'] = '%Y%m%d%H%M%S'
|
245
|
+
elsif conf['path'].index('%M')
|
246
|
+
conf['time_slice_format'] = '%Y%m%d%H%M'
|
247
|
+
elsif conf['path'].index('%H')
|
248
|
+
conf['time_slice_format'] = '%Y%m%d%H'
|
249
|
+
end
|
250
|
+
end
|
251
|
+
if pos = (conf['path'] || '').index('*')
|
252
|
+
@path_prefix = conf['path'][0,pos]
|
253
|
+
@path_suffix = conf['path'][pos+1..-1]
|
254
|
+
conf['buffer_path'] ||= "#{conf['path']}"
|
255
|
+
elsif (conf['path'] || '%Y') !~ /%Y|%m|%d|%H|%M|%S/
|
256
|
+
@path_prefix = conf['path'] + "."
|
257
|
+
@path_suffix = ".log"
|
258
|
+
conf['buffer_path'] ||= "#{conf['path']}.*"
|
259
|
+
elsif (conf['path'] || '') =~ /%Y|%m|%d|%H|%M|%S/
|
260
|
+
conf['buffer_path'] ||= conf['path'].gsub('%Y','yyyy').gsub('%m','mm').gsub('%d','dd').gsub('%H','HH').gsub('%M','MM').gsub('%S','SS')
|
261
|
+
end
|
262
|
+
|
263
|
+
super
|
264
|
+
|
265
|
+
unless @path.index('/') == 0
|
266
|
+
raise Fluent::ConfigError, "Path on filesystem MUST starts with '/', but '#{@path}'"
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
def start
|
271
|
+
super
|
272
|
+
# init
|
273
|
+
end
|
274
|
+
|
275
|
+
def shutdown
|
276
|
+
super
|
277
|
+
# destroy
|
278
|
+
end
|
279
|
+
|
280
|
+
def record_to_string(record)
|
281
|
+
record.to_json
|
282
|
+
end
|
283
|
+
|
284
|
+
def format(tag, time, record)
|
285
|
+
time_str = @timef.format(time)
|
286
|
+
time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
|
287
|
+
end
|
288
|
+
|
289
|
+
def path_format(chunk_key)
|
290
|
+
suffix = case @compress
|
291
|
+
when :gz
|
292
|
+
'.gz'
|
293
|
+
else
|
294
|
+
''
|
295
|
+
end
|
296
|
+
if @path_prefix and @path_suffix
|
297
|
+
if @compress
|
298
|
+
i = 0
|
299
|
+
begin
|
300
|
+
path = "#{@path_prefix}#{chunk_key}_#{i}#{@path_suffix}#{suffix}"
|
301
|
+
i += 1
|
302
|
+
end while File.exist?(path)
|
303
|
+
path
|
304
|
+
else
|
305
|
+
"#{@path_prefix}#{chunk_key}_#{@path_suffix}#{suffix}"
|
306
|
+
end
|
307
|
+
else
|
308
|
+
if @compress
|
309
|
+
path_base = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
|
310
|
+
path = path_base + suffix
|
311
|
+
if File.exist?(path)
|
312
|
+
i = 0
|
313
|
+
begin
|
314
|
+
path = "#{path_base}.#{i}#{suffix}"
|
315
|
+
i += 1
|
316
|
+
end while File.exist?(path)
|
317
|
+
end
|
318
|
+
path
|
319
|
+
else
|
320
|
+
Time.strptime(chunk_key, @time_slice_format).strftime(@path)
|
321
|
+
end
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
def write(chunk)
|
326
|
+
path = path_format(chunk.key)
|
327
|
+
|
328
|
+
begin
|
329
|
+
FileUtils.mkdir_p File.dirname(path)
|
330
|
+
|
331
|
+
case @compress
|
332
|
+
when :gz
|
333
|
+
Zlib::GzipWriter.open(path) {|f|
|
334
|
+
chunk.write_to(f)
|
335
|
+
}
|
336
|
+
else
|
337
|
+
File.open(path, "a") {|f|
|
338
|
+
chunk.write_to(f)
|
339
|
+
}
|
340
|
+
end
|
341
|
+
rescue
|
342
|
+
$log.error "failed to write data: path #{path}"
|
343
|
+
raise
|
344
|
+
end
|
345
|
+
path
|
346
|
+
end
|
347
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
unless ENV.has_key?('VERBOSE')
|
16
|
+
nulllogger = Object.new
|
17
|
+
nulllogger.instance_eval {|obj|
|
18
|
+
def method_missing(method, *args)
|
19
|
+
# pass
|
20
|
+
end
|
21
|
+
}
|
22
|
+
$log = nulllogger
|
23
|
+
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_file_alternative'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class FileAlternativeOutputTest < Test::Unit::TestCase
|
4
|
+
TMP_DIR = File.dirname(__FILE__) + "/../tmp"
|
5
|
+
|
6
|
+
CONFIG = %[
|
7
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d
|
8
|
+
compress gz
|
9
|
+
]
|
10
|
+
|
11
|
+
def setup
|
12
|
+
Fluent::Test.setup
|
13
|
+
FileUtils.rm_rf(TMP_DIR)
|
14
|
+
FileUtils.mkdir_p(TMP_DIR)
|
15
|
+
end
|
16
|
+
|
17
|
+
def create_driver(conf = CONFIG, tag='test')
|
18
|
+
Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::FileAlternativeOutput, tag).configure(conf)
|
19
|
+
end
|
20
|
+
|
21
|
+
# config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
|
22
|
+
|
23
|
+
# path
|
24
|
+
# output_include_time , output_include_tag , output_data_type
|
25
|
+
# add_newline , field_separator
|
26
|
+
# remove_prefix , :default_tag
|
27
|
+
def test_configure
|
28
|
+
# many many many tests should be written, for PlainTextFormatterMixin ...
|
29
|
+
|
30
|
+
d = create_driver %[
|
31
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
32
|
+
]
|
33
|
+
assert_equal '%Y%m%d%H%M%S', d.instance.time_slice_format
|
34
|
+
assert_nil d.instance.compress
|
35
|
+
assert_equal true, d.instance.output_include_time
|
36
|
+
assert_equal true, d.instance.output_include_tag
|
37
|
+
assert_equal 'json', d.instance.output_data_type
|
38
|
+
assert_equal true, d.instance.add_newline
|
39
|
+
assert_equal "\t", d.instance.field_separator
|
40
|
+
assert_nil d.instance.remove_prefix
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_format
|
44
|
+
d1 = create_driver
|
45
|
+
|
46
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
47
|
+
d1.emit({"a"=>1}, time)
|
48
|
+
d1.emit({"a"=>2}, time)
|
49
|
+
d1.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
|
50
|
+
d1.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
51
|
+
d1.run
|
52
|
+
|
53
|
+
d2 = create_driver %[
|
54
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
55
|
+
output_include_time false
|
56
|
+
output_include_tag false
|
57
|
+
output_data_type attr:message
|
58
|
+
]
|
59
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
60
|
+
d2.emit({"message" => "abc-xyz-123", "other" => "zzz"}, time)
|
61
|
+
d2.emit({"message" => "123-456-789", "other" => "ppp"}, time)
|
62
|
+
d2.expect_format %[abc-xyz-123\n]
|
63
|
+
d2.expect_format %[123-456-789\n]
|
64
|
+
d2.run
|
65
|
+
|
66
|
+
d3 = create_driver %[
|
67
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
68
|
+
output_include_time false
|
69
|
+
output_include_tag false
|
70
|
+
output_data_type attr:server,level,log
|
71
|
+
field_separator COMMA
|
72
|
+
add_newline false
|
73
|
+
]
|
74
|
+
assert_equal ',', d3.instance.f_separator
|
75
|
+
|
76
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
77
|
+
d3.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
|
78
|
+
d3.emit({"server" => "app01", "level" => "info", "log" => "Send response\n"}, time)
|
79
|
+
d3.expect_format %[www01,warn,Exception\n]
|
80
|
+
d3.expect_format %[app01,info,Send response\n]
|
81
|
+
d3.run
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_write
|
85
|
+
d2 = create_driver %[
|
86
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
87
|
+
output_include_time false
|
88
|
+
output_include_tag false
|
89
|
+
output_data_type attr:message
|
90
|
+
utc
|
91
|
+
]
|
92
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
93
|
+
d2.emit({"message" => "abc-xyz-123", "other" => "zzz"}, time)
|
94
|
+
d2.emit({"message" => "123-456-789", "other" => "ppp"}, time)
|
95
|
+
d2.expect_format %[abc-xyz-123\n]
|
96
|
+
d2.expect_format %[123-456-789\n]
|
97
|
+
path = d2.run
|
98
|
+
assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15", path[0]
|
99
|
+
|
100
|
+
d3 = create_driver %[
|
101
|
+
path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
|
102
|
+
compress gzip
|
103
|
+
output_include_time false
|
104
|
+
output_include_tag false
|
105
|
+
output_data_type attr:server,level,log
|
106
|
+
field_separator COMMA
|
107
|
+
add_newline false
|
108
|
+
utc
|
109
|
+
]
|
110
|
+
assert_equal ',', d3.instance.f_separator
|
111
|
+
|
112
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
113
|
+
d3.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
|
114
|
+
d3.emit({"server" => "app01", "level" => "info", "log" => "Send response\n"}, time)
|
115
|
+
d3.expect_format %[www01,warn,Exception\n]
|
116
|
+
d3.expect_format %[app01,info,Send response\n]
|
117
|
+
path = d3.run
|
118
|
+
assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15.gz", path[0]
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'time'
|
4
|
+
|
5
|
+
class FileOutputTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
Fluent::Test.setup
|
8
|
+
FileUtils.rm_rf(TMP_DIR)
|
9
|
+
FileUtils.mkdir_p(TMP_DIR)
|
10
|
+
end
|
11
|
+
|
12
|
+
TMP_DIR = File.dirname(__FILE__) + "/../tmp"
|
13
|
+
|
14
|
+
CONFIG = %[
|
15
|
+
path #{TMP_DIR}/out_file_test
|
16
|
+
compress gz
|
17
|
+
utc
|
18
|
+
]
|
19
|
+
|
20
|
+
def create_driver(conf = CONFIG)
|
21
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_configure
|
25
|
+
d = create_driver %[
|
26
|
+
path test_path
|
27
|
+
compress gz
|
28
|
+
]
|
29
|
+
assert_equal 'test_path', d.instance.path
|
30
|
+
assert_equal :gz, d.instance.compress
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_format
|
34
|
+
d = create_driver
|
35
|
+
|
36
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
37
|
+
d.emit({"a"=>1}, time)
|
38
|
+
d.emit({"a"=>2}, time)
|
39
|
+
|
40
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
|
41
|
+
d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
42
|
+
|
43
|
+
d.run
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_write
|
47
|
+
d = create_driver
|
48
|
+
|
49
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
50
|
+
d.emit({"a"=>1}, time)
|
51
|
+
d.emit({"a"=>2}, time)
|
52
|
+
|
53
|
+
# FileOutput#write returns path
|
54
|
+
path = d.run
|
55
|
+
expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
|
56
|
+
assert_equal expect_path, path
|
57
|
+
|
58
|
+
data = Zlib::GzipReader.open(expect_path) {|f| f.read }
|
59
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
60
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
61
|
+
data
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_write_path_increment
|
65
|
+
d = create_driver
|
66
|
+
|
67
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
68
|
+
d.emit({"a"=>1}, time)
|
69
|
+
d.emit({"a"=>2}, time)
|
70
|
+
|
71
|
+
# FileOutput#write returns path
|
72
|
+
path = d.run
|
73
|
+
assert_equal "#{TMP_DIR}/out_file_test._0.log.gz", path
|
74
|
+
|
75
|
+
path = d.run
|
76
|
+
assert_equal "#{TMP_DIR}/out_file_test._1.log.gz", path
|
77
|
+
|
78
|
+
path = d.run
|
79
|
+
assert_equal "#{TMP_DIR}/out_file_test._2.log.gz", path
|
80
|
+
end
|
81
|
+
end
|
metadata
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-file-alternative
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- TAGOMORI Satoshi
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-03-16 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: &2168521620 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *2168521620
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: fluentd
|
27
|
+
requirement: &2168521180 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *2168521180
|
36
|
+
description: alternative implementation of out_file, with various configurations
|
37
|
+
email:
|
38
|
+
- tagomoris@gmail.com
|
39
|
+
executables: []
|
40
|
+
extensions: []
|
41
|
+
extra_rdoc_files: []
|
42
|
+
files:
|
43
|
+
- .gitignore
|
44
|
+
- Gemfile
|
45
|
+
- LICENSE.txt
|
46
|
+
- README.md
|
47
|
+
- Rakefile
|
48
|
+
- fluent-plugin-file-alternative.gemspec
|
49
|
+
- lib/fluent/plugin/out_file_alternative.rb
|
50
|
+
- test/helper.rb
|
51
|
+
- test/plugin/test_out_file_alternative.rb
|
52
|
+
- test/plugin/test_out_file_original.rb
|
53
|
+
homepage: https://github.com/tagomoris/fluent-plugin-file-alternative
|
54
|
+
licenses: []
|
55
|
+
post_install_message:
|
56
|
+
rdoc_options: []
|
57
|
+
require_paths:
|
58
|
+
- lib
|
59
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
60
|
+
none: false
|
61
|
+
requirements:
|
62
|
+
- - ! '>='
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: '0'
|
65
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ! '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
requirements: []
|
72
|
+
rubyforge_project:
|
73
|
+
rubygems_version: 1.8.6
|
74
|
+
signing_key:
|
75
|
+
specification_version: 3
|
76
|
+
summary: alternative implementation of out_file
|
77
|
+
test_files:
|
78
|
+
- test/helper.rb
|
79
|
+
- test/plugin/test_out_file_alternative.rb
|
80
|
+
- test/plugin/test_out_file_original.rb
|