fluent-plugin-file-alternative 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,25 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+
19
+ # For TextMate, emacs, vim
20
+ *.tmproj
21
+ tmtags
22
+ *~
23
+ \#*
24
+ .\#*
25
+ *.swp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-file-alternative.gemspec
4
+ gemspec
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012- TAGOMORI Satoshi
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
@@ -0,0 +1,71 @@
1
+ # fluent-plugin-file-alternative
2
+
3
+ File output plugin alternative implementation, **is 100% compatible with fluentd built-in 'out_file'**, and added many options to format output as you want.
4
+
5
+ FileAlternativeOutput slices data by time (for specified units), and store these data as plain text on hdfs. You can specify to:
6
+
7
+ * format whole data as serialized JSON, single attribute or separated multi attributes
8
+ * include time as line header, or not
9
+ * include tag as line header, or not
10
+ * change field separator (default: TAB)
11
+ * add new line as termination, or not
12
+
13
+ And you can specify output file path as:
14
+
15
+ * Standard out_file way
16
+ * configure 'path /path/to/dir/access'
17
+ * and 'time\_slice\_format %Y%m%d'
18
+ * got '/path/to/dir/access.20120316.log'
19
+ * Alternative style
20
+ * configure 'path /path/to/dir/access.%Y%m%d.log' only
21
+ * got '/path/to/dir/access.20120316.log'
22
+
23
+ And, gzip compression is also supported.
24
+
25
+ ## Configuration
26
+
27
+ ### FileAlternativeOutput
28
+
29
+ Standard out_file way (hourly log, compression, time-tag-json):
30
+
31
+ <match out.**>
32
+ type file\_alternative
33
+ path /var/log/service/access.*.log
34
+ time\_slice\_output %Y%m%d_%H
35
+ compress gzip
36
+ </match>
37
+
38
+ By this configuration, in gzip compressed file '/var/log/service/access.20120316_23.log.gz', you get:
39
+
40
+ 2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
41
+ 2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
42
+ 2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
43
+ 2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
44
+
45
+ If you don't want fluentd-time and tag in written file, and messages with single attribute (as raw full apache log with newline):
46
+
47
+ <match out.**>
48
+ type file\_alternative
49
+ path /var/log/service/access.%Y%m%d_%H.log
50
+ compress gzip
51
+ output_include_time false
52
+ output_include_tag false
53
+ output_data_type attr:message
54
+ add_newline false
55
+ </match>
56
+
57
+ Then, you will get:
58
+
59
+ 192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
60
+ 192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
61
+ 192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
62
+
63
+ ## TODO
64
+
65
+ * consider what to do next
66
+ * patches welcome!
67
+
68
+ ## Copyright
69
+
70
+ Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
71
+ License:: Apache License, Version 2.0
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
@@ -0,0 +1,20 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ Gem::Specification.new do |gem|
4
+ gem.name = "fluent-plugin-file-alternative"
5
+ gem.version = "0.1.0"
6
+
7
+ gem.authors = ["TAGOMORI Satoshi"]
8
+ gem.email = ["tagomoris@gmail.com"]
9
+ gem.description = %q{alternative implementation of out_file, with various configurations}
10
+ gem.summary = %q{alternative implementation of out_file}
11
+ gem.homepage = "https://github.com/tagomoris/fluent-plugin-file-alternative"
12
+
13
+ gem.files = `git ls-files`.split($\)
14
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
15
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
16
+ gem.require_paths = ["lib"]
17
+
18
+ gem.add_development_dependency "fluentd"
19
+ gem.add_runtime_dependency "fluentd"
20
+ end
@@ -0,0 +1,347 @@
1
+ module FluentExt; end
2
+ module FluentExt::PlainTextFormatterMixin
3
+ # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
4
+
5
+ attr_accessor :output_include_time, :output_include_tag, :output_data_type
6
+ attr_accessor :add_newline, :field_separator
7
+ attr_accessor :remove_prefix, :default_tag
8
+
9
+ attr_accessor :f_separator
10
+
11
+ def configure(conf)
12
+ super
13
+
14
+ @output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
15
+ @output_include_time = true if @output_include_time.nil?
16
+
17
+ @output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
18
+ @output_include_tag = true if @output_include_tag.nil?
19
+
20
+ @output_data_type = conf['output_data_type']
21
+ @output_data_type = 'json' if @output_data_type.nil?
22
+
23
+ @f_separator = case conf['field_separator']
24
+ when 'SPACE' then ' '
25
+ when 'COMMA' then ','
26
+ else "\t"
27
+ end
28
+ @add_newline = Fluent::Config.bool_value(conf['add_newline'])
29
+ if @add_newline.nil?
30
+ @add_newline = true
31
+ end
32
+
33
+ @remove_prefix = conf['remove_prefix']
34
+ if @remove_prefix
35
+ @removed_prefix_string = @remove_prefix + '.'
36
+ @removed_length = @removed_prefix_string.length
37
+ end
38
+ if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
39
+ @default_tag = conf['default_tag']
40
+ if @default_tag.nil? or @default_tag.length < 1
41
+ raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
42
+ end
43
+ end
44
+
45
+ # default timezone: utc
46
+ if conf['localtime'].nil? and conf['utc'].nil?
47
+ @utc = true
48
+ @localtime = false
49
+ elsif not @localtime and not @utc
50
+ @utc = true
51
+ @localtime = false
52
+ end
53
+ # mix-in default time formatter (or you can overwrite @timef on your own configure)
54
+ @timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
55
+
56
+ @custom_attributes = []
57
+ if @output_data_type == 'json'
58
+ self.instance_eval {
59
+ def stringify_record(record)
60
+ record.to_json
61
+ end
62
+ }
63
+ elsif @output_data_type =~ /^attr:(.*)$/
64
+ @custom_attributes = $1.split(',')
65
+ if @custom_attributes.size > 1
66
+ self.instance_eval {
67
+ def stringify_record(record)
68
+ @custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@f_separator)
69
+ end
70
+ }
71
+ elsif @custom_attributes.size == 1
72
+ self.instance_eval {
73
+ def stringify_record(record)
74
+ (record[@custom_attributes[0]] || 'NULL').to_s
75
+ end
76
+ }
77
+ else
78
+ raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
79
+ end
80
+ else
81
+ raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
82
+ end
83
+
84
+ if @output_include_time and @output_include_tag
85
+ if @add_newline and @remove_prefix
86
+ self.instance_eval {
87
+ def format(tag,time,record)
88
+ if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
89
+ tag == @remove_prefix
90
+ tag = tag[@removed_length..-1] || @default_tag
91
+ end
92
+ @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
93
+ end
94
+ }
95
+ elsif @add_newline
96
+ self.instance_eval {
97
+ def format(tag,time,record)
98
+ @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
99
+ end
100
+ }
101
+ elsif @remove_prefix
102
+ self.instance_eval {
103
+ def format(tag,time,record)
104
+ if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
105
+ tag == @remove_prefix
106
+ tag = tag[@removed_length..-1] || @default_tag
107
+ end
108
+ @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
109
+ end
110
+ }
111
+ else
112
+ self.instance_eval {
113
+ def format(tag,time,record)
114
+ @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
115
+ end
116
+ }
117
+ end
118
+ elsif @output_include_time
119
+ if @add_newline
120
+ self.instance_eval {
121
+ def format(tag,time,record);
122
+ @timef.format(time) + @f_separator + stringify_record(record) + "\n"
123
+ end
124
+ }
125
+ else
126
+ self.instance_eval {
127
+ def format(tag,time,record);
128
+ @timef.format(time) + @f_separator + stringify_record(record)
129
+ end
130
+ }
131
+ end
132
+ elsif @output_include_tag
133
+ if @add_newline and @remove_prefix
134
+ self.instance_eval {
135
+ def format(tag,time,record)
136
+ if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
137
+ tag == @remove_prefix
138
+ tag = tag[@removed_length..-1] || @default_tag
139
+ end
140
+ tag + @f_separator + stringify_record(record) + "\n"
141
+ end
142
+ }
143
+ elsif @add_newline
144
+ self.instance_eval {
145
+ def format(tag,time,record)
146
+ tag + @f_separator + stringify_record(record) + "\n"
147
+ end
148
+ }
149
+ elsif @remove_prefix
150
+ self.instance_eval {
151
+ def format(tag,time,record)
152
+ if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
153
+ tag == @remove_prefix
154
+ tag = tag[@removed_length..-1] || @default_tag
155
+ end
156
+ tag + @f_separator + stringify_record(record)
157
+ end
158
+ }
159
+ else
160
+ self.instance_eval {
161
+ def format(tag,time,record)
162
+ tag + @f_separator + stringify_record(record)
163
+ end
164
+ }
165
+ end
166
+ else # without time, tag
167
+ if @add_newline
168
+ self.instance_eval {
169
+ def format(tag,time,record);
170
+ stringify_record(record) + "\n"
171
+ end
172
+ }
173
+ else
174
+ self.instance_eval {
175
+ def format(tag,time,record);
176
+ stringify_record(record)
177
+ end
178
+ }
179
+ end
180
+ end
181
+ end
182
+
183
+ def stringify_record(record)
184
+ record.to_json
185
+ end
186
+
187
+ def format(tag, time, record)
188
+ if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
189
+ tag = tag[@removed_length..-1] || @default_tag
190
+ end
191
+ time_str = if @output_include_time
192
+ @timef.format(time) + @f_separator
193
+ else
194
+ ''
195
+ end
196
+ tag_str = if @output_include_tag
197
+ tag + @f_separator
198
+ else
199
+ ''
200
+ end
201
+ time_str + tag_str + stringify_record(record) + "\n"
202
+ end
203
+
204
+ end
205
+
206
+ class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
207
+ Fluent::Plugin.register_output('file_alternative', self)
208
+
209
+ SUPPORTED_COMPRESS = {
210
+ :gz => :gz,
211
+ :gzip => :gz,
212
+ }
213
+
214
+ config_set_default :time_slice_format, '%Y%m%d' # %Y%m%d%H
215
+
216
+ config_param :path, :string # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S
217
+
218
+ config_param :compress, :default => nil do |val|
219
+ c = SUPPORTED_COMPRESS[val.to_sym]
220
+ unless c
221
+ raise ConfigError, "Unsupported compression algorithm '#{compress}'"
222
+ end
223
+ c
224
+ end
225
+
226
+ include FluentExt::PlainTextFormatterMixin
227
+
228
+ config_set_default :output_include_time, true
229
+ config_set_default :output_include_tag, true
230
+ config_set_default :output_data_type, 'json'
231
+ config_set_default :field_separator, "\t"
232
+ config_set_default :add_newline, true
233
+ config_set_default :remove_prefix, nil
234
+
235
+ def initialize
236
+ super
237
+ require 'time'
238
+ require 'zlib'
239
+ end
240
+
241
+ def configure(conf)
242
+ if conf['path']
243
+ if conf['path'].index('%S')
244
+ conf['time_slice_format'] = '%Y%m%d%H%M%S'
245
+ elsif conf['path'].index('%M')
246
+ conf['time_slice_format'] = '%Y%m%d%H%M'
247
+ elsif conf['path'].index('%H')
248
+ conf['time_slice_format'] = '%Y%m%d%H'
249
+ end
250
+ end
251
+ if pos = (conf['path'] || '').index('*')
252
+ @path_prefix = conf['path'][0,pos]
253
+ @path_suffix = conf['path'][pos+1..-1]
254
+ conf['buffer_path'] ||= "#{conf['path']}"
255
+ elsif (conf['path'] || '%Y') !~ /%Y|%m|%d|%H|%M|%S/
256
+ @path_prefix = conf['path'] + "."
257
+ @path_suffix = ".log"
258
+ conf['buffer_path'] ||= "#{conf['path']}.*"
259
+ elsif (conf['path'] || '') =~ /%Y|%m|%d|%H|%M|%S/
260
+ conf['buffer_path'] ||= conf['path'].gsub('%Y','yyyy').gsub('%m','mm').gsub('%d','dd').gsub('%H','HH').gsub('%M','MM').gsub('%S','SS')
261
+ end
262
+
263
+ super
264
+
265
+ unless @path.index('/') == 0
266
+ raise Fluent::ConfigError, "Path on filesystem MUST starts with '/', but '#{@path}'"
267
+ end
268
+ end
269
+
270
+ def start
271
+ super
272
+ # init
273
+ end
274
+
275
+ def shutdown
276
+ super
277
+ # destroy
278
+ end
279
+
280
+ def record_to_string(record)
281
+ record.to_json
282
+ end
283
+
284
+ def format(tag, time, record)
285
+ time_str = @timef.format(time)
286
+ time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
287
+ end
288
+
289
+ def path_format(chunk_key)
290
+ suffix = case @compress
291
+ when :gz
292
+ '.gz'
293
+ else
294
+ ''
295
+ end
296
+ if @path_prefix and @path_suffix
297
+ if @compress
298
+ i = 0
299
+ begin
300
+ path = "#{@path_prefix}#{chunk_key}_#{i}#{@path_suffix}#{suffix}"
301
+ i += 1
302
+ end while File.exist?(path)
303
+ path
304
+ else
305
+ "#{@path_prefix}#{chunk_key}_#{@path_suffix}#{suffix}"
306
+ end
307
+ else
308
+ if @compress
309
+ path_base = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
310
+ path = path_base + suffix
311
+ if File.exist?(path)
312
+ i = 0
313
+ begin
314
+ path = "#{path_base}.#{i}#{suffix}"
315
+ i += 1
316
+ end while File.exist?(path)
317
+ end
318
+ path
319
+ else
320
+ Time.strptime(chunk_key, @time_slice_format).strftime(@path)
321
+ end
322
+ end
323
+ end
324
+
325
+ def write(chunk)
326
+ path = path_format(chunk.key)
327
+
328
+ begin
329
+ FileUtils.mkdir_p File.dirname(path)
330
+
331
+ case @compress
332
+ when :gz
333
+ Zlib::GzipWriter.open(path) {|f|
334
+ chunk.write_to(f)
335
+ }
336
+ else
337
+ File.open(path, "a") {|f|
338
+ chunk.write_to(f)
339
+ }
340
+ end
341
+ rescue
342
+ $log.error "failed to write data: path #{path}"
343
+ raise
344
+ end
345
+ path
346
+ end
347
+ end
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_file_alternative'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,120 @@
1
+ require 'helper'
2
+
3
+ class FileAlternativeOutputTest < Test::Unit::TestCase
4
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp"
5
+
6
+ CONFIG = %[
7
+ path #{TMP_DIR}/accesslog.%Y-%m-%d
8
+ compress gz
9
+ ]
10
+
11
+ def setup
12
+ Fluent::Test.setup
13
+ FileUtils.rm_rf(TMP_DIR)
14
+ FileUtils.mkdir_p(TMP_DIR)
15
+ end
16
+
17
+ def create_driver(conf = CONFIG, tag='test')
18
+ Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::FileAlternativeOutput, tag).configure(conf)
19
+ end
20
+
21
+ # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
22
+
23
+ # path
24
+ # output_include_time , output_include_tag , output_data_type
25
+ # add_newline , field_separator
26
+ # remove_prefix , :default_tag
27
+ def test_configure
28
+ # many many many tests should be written, for PlainTextFormatterMixin ...
29
+
30
+ d = create_driver %[
31
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
32
+ ]
33
+ assert_equal '%Y%m%d%H%M%S', d.instance.time_slice_format
34
+ assert_nil d.instance.compress
35
+ assert_equal true, d.instance.output_include_time
36
+ assert_equal true, d.instance.output_include_tag
37
+ assert_equal 'json', d.instance.output_data_type
38
+ assert_equal true, d.instance.add_newline
39
+ assert_equal "\t", d.instance.field_separator
40
+ assert_nil d.instance.remove_prefix
41
+ end
42
+
43
+ def test_format
44
+ d1 = create_driver
45
+
46
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
47
+ d1.emit({"a"=>1}, time)
48
+ d1.emit({"a"=>2}, time)
49
+ d1.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
50
+ d1.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
51
+ d1.run
52
+
53
+ d2 = create_driver %[
54
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
55
+ output_include_time false
56
+ output_include_tag false
57
+ output_data_type attr:message
58
+ ]
59
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
60
+ d2.emit({"message" => "abc-xyz-123", "other" => "zzz"}, time)
61
+ d2.emit({"message" => "123-456-789", "other" => "ppp"}, time)
62
+ d2.expect_format %[abc-xyz-123\n]
63
+ d2.expect_format %[123-456-789\n]
64
+ d2.run
65
+
66
+ d3 = create_driver %[
67
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
68
+ output_include_time false
69
+ output_include_tag false
70
+ output_data_type attr:server,level,log
71
+ field_separator COMMA
72
+ add_newline false
73
+ ]
74
+ assert_equal ',', d3.instance.f_separator
75
+
76
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
77
+ d3.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
78
+ d3.emit({"server" => "app01", "level" => "info", "log" => "Send response\n"}, time)
79
+ d3.expect_format %[www01,warn,Exception\n]
80
+ d3.expect_format %[app01,info,Send response\n]
81
+ d3.run
82
+ end
83
+
84
+ def test_write
85
+ d2 = create_driver %[
86
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
87
+ output_include_time false
88
+ output_include_tag false
89
+ output_data_type attr:message
90
+ utc
91
+ ]
92
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
93
+ d2.emit({"message" => "abc-xyz-123", "other" => "zzz"}, time)
94
+ d2.emit({"message" => "123-456-789", "other" => "ppp"}, time)
95
+ d2.expect_format %[abc-xyz-123\n]
96
+ d2.expect_format %[123-456-789\n]
97
+ path = d2.run
98
+ assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15", path[0]
99
+
100
+ d3 = create_driver %[
101
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
102
+ compress gzip
103
+ output_include_time false
104
+ output_include_tag false
105
+ output_data_type attr:server,level,log
106
+ field_separator COMMA
107
+ add_newline false
108
+ utc
109
+ ]
110
+ assert_equal ',', d3.instance.f_separator
111
+
112
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
113
+ d3.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
114
+ d3.emit({"server" => "app01", "level" => "info", "log" => "Send response\n"}, time)
115
+ d3.expect_format %[www01,warn,Exception\n]
116
+ d3.expect_format %[app01,info,Send response\n]
117
+ path = d3.run
118
+ assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15.gz", path[0]
119
+ end
120
+ end
@@ -0,0 +1,81 @@
1
+ require 'fluent/test'
2
+ require 'fileutils'
3
+ require 'time'
4
+
5
+ class FileOutputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ FileUtils.rm_rf(TMP_DIR)
9
+ FileUtils.mkdir_p(TMP_DIR)
10
+ end
11
+
12
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp"
13
+
14
+ CONFIG = %[
15
+ path #{TMP_DIR}/out_file_test
16
+ compress gz
17
+ utc
18
+ ]
19
+
20
+ def create_driver(conf = CONFIG)
21
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
22
+ end
23
+
24
+ def test_configure
25
+ d = create_driver %[
26
+ path test_path
27
+ compress gz
28
+ ]
29
+ assert_equal 'test_path', d.instance.path
30
+ assert_equal :gz, d.instance.compress
31
+ end
32
+
33
+ def test_format
34
+ d = create_driver
35
+
36
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
37
+ d.emit({"a"=>1}, time)
38
+ d.emit({"a"=>2}, time)
39
+
40
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
41
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
42
+
43
+ d.run
44
+ end
45
+
46
+ def test_write
47
+ d = create_driver
48
+
49
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
50
+ d.emit({"a"=>1}, time)
51
+ d.emit({"a"=>2}, time)
52
+
53
+ # FileOutput#write returns path
54
+ path = d.run
55
+ expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
56
+ assert_equal expect_path, path
57
+
58
+ data = Zlib::GzipReader.open(expect_path) {|f| f.read }
59
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
60
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
61
+ data
62
+ end
63
+
64
+ def test_write_path_increment
65
+ d = create_driver
66
+
67
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
68
+ d.emit({"a"=>1}, time)
69
+ d.emit({"a"=>2}, time)
70
+
71
+ # FileOutput#write returns path
72
+ path = d.run
73
+ assert_equal "#{TMP_DIR}/out_file_test._0.log.gz", path
74
+
75
+ path = d.run
76
+ assert_equal "#{TMP_DIR}/out_file_test._1.log.gz", path
77
+
78
+ path = d.run
79
+ assert_equal "#{TMP_DIR}/out_file_test._2.log.gz", path
80
+ end
81
+ end
metadata ADDED
@@ -0,0 +1,80 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-file-alternative
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - TAGOMORI Satoshi
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-03-16 00:00:00.000000000Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: &2168521620 !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: *2168521620
25
+ - !ruby/object:Gem::Dependency
26
+ name: fluentd
27
+ requirement: &2168521180 !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: *2168521180
36
+ description: alternative implementation of out_file, with various configurations
37
+ email:
38
+ - tagomoris@gmail.com
39
+ executables: []
40
+ extensions: []
41
+ extra_rdoc_files: []
42
+ files:
43
+ - .gitignore
44
+ - Gemfile
45
+ - LICENSE.txt
46
+ - README.md
47
+ - Rakefile
48
+ - fluent-plugin-file-alternative.gemspec
49
+ - lib/fluent/plugin/out_file_alternative.rb
50
+ - test/helper.rb
51
+ - test/plugin/test_out_file_alternative.rb
52
+ - test/plugin/test_out_file_original.rb
53
+ homepage: https://github.com/tagomoris/fluent-plugin-file-alternative
54
+ licenses: []
55
+ post_install_message:
56
+ rdoc_options: []
57
+ require_paths:
58
+ - lib
59
+ required_ruby_version: !ruby/object:Gem::Requirement
60
+ none: false
61
+ requirements:
62
+ - - ! '>='
63
+ - !ruby/object:Gem::Version
64
+ version: '0'
65
+ required_rubygems_version: !ruby/object:Gem::Requirement
66
+ none: false
67
+ requirements:
68
+ - - ! '>='
69
+ - !ruby/object:Gem::Version
70
+ version: '0'
71
+ requirements: []
72
+ rubyforge_project:
73
+ rubygems_version: 1.8.6
74
+ signing_key:
75
+ specification_version: 3
76
+ summary: alternative implementation of out_file
77
+ test_files:
78
+ - test/helper.rb
79
+ - test/plugin/test_out_file_alternative.rb
80
+ - test/plugin/test_out_file_original.rb