fluent-plugin-file-alternative 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,25 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+
19
+ # For TextMate, emacs, vim
20
+ *.tmproj
21
+ tmtags
22
+ *~
23
+ \#*
24
+ .\#*
25
+ *.swp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-file-alternative.gemspec
4
+ gemspec
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012- TAGOMORI Satoshi
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
@@ -0,0 +1,71 @@
1
+ # fluent-plugin-file-alternative
2
+
3
+ File output plugin alternative implementation, **is 100% compatible with fluentd built-in 'out_file'**, and added many options to format output as you want.
4
+
5
+ FileAlternativeOutput slices data by time (for specified units), and store these data as plain text on hdfs. You can specify to:
6
+
7
+ * format whole data as serialized JSON, single attribute or separated multi attributes
8
+ * include time as line header, or not
9
+ * include tag as line header, or not
10
+ * change field separator (default: TAB)
11
+ * add new line as termination, or not
12
+
13
+ And you can specify output file path as:
14
+
15
+ * Standard out_file way
16
+ * configure 'path /path/to/dir/access'
17
+ * and 'time\_slice\_format %Y%m%d'
18
+ * got '/path/to/dir/access.20120316.log'
19
+ * Alternative style
20
+ * configure 'path /path/to/dir/access.%Y%m%d.log' only
21
+ * got '/path/to/dir/access.20120316.log'
22
+
23
+ And, gzip compression is also supported.
24
+
25
+ ## Configuration
26
+
27
+ ### FileAlternativeOutput
28
+
29
+ Standard out_file way (hourly log, compression, time-tag-json):
30
+
31
+ <match out.**>
32
+ type file\_alternative
33
+ path /var/log/service/access.*.log
34
+ time\_slice\_output %Y%m%d_%H
35
+ compress gzip
36
+ </match>
37
+
38
+ By this configuration, in gzip compressed file '/var/log/service/access.20120316_23.log.gz', you get:
39
+
40
+ 2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
41
+ 2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
42
+ 2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
43
+ 2012-03-16T23:59:40 [TAB] out.service.xxx [TAB] {"field1":"value1","field2":"value2"}
44
+
45
+ If you don't want fluentd-time and tag in written file, and messages with single attribute (as raw full apache log with newline):
46
+
47
+ <match out.**>
48
+ type file\_alternative
49
+ path /var/log/service/access.%Y%m%d_%H.log
50
+ compress gzip
51
+ output_include_time false
52
+ output_include_tag false
53
+ output_data_type attr:message
54
+ add_newline false
55
+ </match>
56
+
57
+ Then, you will get:
58
+
59
+ 192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
60
+ 192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
61
+ 192.168.0.1 - - [16/Mar/2012:23:59:40 +0900] "GET /content/x HTTP/1.1" 200 -
62
+
63
+ ## TODO
64
+
65
+ * consider what to do next
66
+ * patches welcome!
67
+
68
+ ## Copyright
69
+
70
+ Copyright:: Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
71
+ License:: Apache License, Version 2.0
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ require 'rake/testtask'
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'lib' << 'test'
7
+ test.pattern = 'test/**/test_*.rb'
8
+ test.verbose = true
9
+ end
10
+
11
+ task :default => :test
@@ -0,0 +1,20 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ Gem::Specification.new do |gem|
4
+ gem.name = "fluent-plugin-file-alternative"
5
+ gem.version = "0.1.0"
6
+
7
+ gem.authors = ["TAGOMORI Satoshi"]
8
+ gem.email = ["tagomoris@gmail.com"]
9
+ gem.description = %q{alternative implementation of out_file, with various configurations}
10
+ gem.summary = %q{alternative implementation of out_file}
11
+ gem.homepage = "https://github.com/tagomoris/fluent-plugin-file-alternative"
12
+
13
+ gem.files = `git ls-files`.split($\)
14
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
15
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
16
+ gem.require_paths = ["lib"]
17
+
18
+ gem.add_development_dependency "fluentd"
19
+ gem.add_runtime_dependency "fluentd"
20
+ end
@@ -0,0 +1,347 @@
1
+ module FluentExt; end
2
+ module FluentExt::PlainTextFormatterMixin
3
+ # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
4
+
5
+ attr_accessor :output_include_time, :output_include_tag, :output_data_type
6
+ attr_accessor :add_newline, :field_separator
7
+ attr_accessor :remove_prefix, :default_tag
8
+
9
+ attr_accessor :f_separator
10
+
11
+ def configure(conf)
12
+ super
13
+
14
+ @output_include_time = Fluent::Config.bool_value(conf['output_include_time'])
15
+ @output_include_time = true if @output_include_time.nil?
16
+
17
+ @output_include_tag = Fluent::Config.bool_value(conf['output_include_tag'])
18
+ @output_include_tag = true if @output_include_tag.nil?
19
+
20
+ @output_data_type = conf['output_data_type']
21
+ @output_data_type = 'json' if @output_data_type.nil?
22
+
23
+ @f_separator = case conf['field_separator']
24
+ when 'SPACE' then ' '
25
+ when 'COMMA' then ','
26
+ else "\t"
27
+ end
28
+ @add_newline = Fluent::Config.bool_value(conf['add_newline'])
29
+ if @add_newline.nil?
30
+ @add_newline = true
31
+ end
32
+
33
+ @remove_prefix = conf['remove_prefix']
34
+ if @remove_prefix
35
+ @removed_prefix_string = @remove_prefix + '.'
36
+ @removed_length = @removed_prefix_string.length
37
+ end
38
+ if @output_include_tag and @remove_prefix and @remove_prefix.length > 0
39
+ @default_tag = conf['default_tag']
40
+ if @default_tag.nil? or @default_tag.length < 1
41
+ raise Fluent::ConfigError, "Missing 'default_tag' with output_include_tag and remove_prefix."
42
+ end
43
+ end
44
+
45
+ # default timezone: utc
46
+ if conf['localtime'].nil? and conf['utc'].nil?
47
+ @utc = true
48
+ @localtime = false
49
+ elsif not @localtime and not @utc
50
+ @utc = true
51
+ @localtime = false
52
+ end
53
+ # mix-in default time formatter (or you can overwrite @timef on your own configure)
54
+ @timef = @output_include_time ? Fluent::TimeFormatter.new(@time_format, @localtime) : nil
55
+
56
+ @custom_attributes = []
57
+ if @output_data_type == 'json'
58
+ self.instance_eval {
59
+ def stringify_record(record)
60
+ record.to_json
61
+ end
62
+ }
63
+ elsif @output_data_type =~ /^attr:(.*)$/
64
+ @custom_attributes = $1.split(',')
65
+ if @custom_attributes.size > 1
66
+ self.instance_eval {
67
+ def stringify_record(record)
68
+ @custom_attributes.map{|attr| (record[attr] || 'NULL').to_s}.join(@f_separator)
69
+ end
70
+ }
71
+ elsif @custom_attributes.size == 1
72
+ self.instance_eval {
73
+ def stringify_record(record)
74
+ (record[@custom_attributes[0]] || 'NULL').to_s
75
+ end
76
+ }
77
+ else
78
+ raise Fluent::ConfigError, "Invalid attributes specification: '#{@output_data_type}', needs one or more attributes."
79
+ end
80
+ else
81
+ raise Fluent::ConfigError, "Invalid output_data_type: '#{@output_data_type}'. specify 'json' or 'attr:ATTRIBUTE_NAME' or 'attr:ATTR1,ATTR2,...'"
82
+ end
83
+
84
+ if @output_include_time and @output_include_tag
85
+ if @add_newline and @remove_prefix
86
+ self.instance_eval {
87
+ def format(tag,time,record)
88
+ if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
89
+ tag == @remove_prefix
90
+ tag = tag[@removed_length..-1] || @default_tag
91
+ end
92
+ @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
93
+ end
94
+ }
95
+ elsif @add_newline
96
+ self.instance_eval {
97
+ def format(tag,time,record)
98
+ @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record) + "\n"
99
+ end
100
+ }
101
+ elsif @remove_prefix
102
+ self.instance_eval {
103
+ def format(tag,time,record)
104
+ if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
105
+ tag == @remove_prefix
106
+ tag = tag[@removed_length..-1] || @default_tag
107
+ end
108
+ @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
109
+ end
110
+ }
111
+ else
112
+ self.instance_eval {
113
+ def format(tag,time,record)
114
+ @timef.format(time) + @f_separator + tag + @f_separator + stringify_record(record)
115
+ end
116
+ }
117
+ end
118
+ elsif @output_include_time
119
+ if @add_newline
120
+ self.instance_eval {
121
+ def format(tag,time,record);
122
+ @timef.format(time) + @f_separator + stringify_record(record) + "\n"
123
+ end
124
+ }
125
+ else
126
+ self.instance_eval {
127
+ def format(tag,time,record);
128
+ @timef.format(time) + @f_separator + stringify_record(record)
129
+ end
130
+ }
131
+ end
132
+ elsif @output_include_tag
133
+ if @add_newline and @remove_prefix
134
+ self.instance_eval {
135
+ def format(tag,time,record)
136
+ if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
137
+ tag == @remove_prefix
138
+ tag = tag[@removed_length..-1] || @default_tag
139
+ end
140
+ tag + @f_separator + stringify_record(record) + "\n"
141
+ end
142
+ }
143
+ elsif @add_newline
144
+ self.instance_eval {
145
+ def format(tag,time,record)
146
+ tag + @f_separator + stringify_record(record) + "\n"
147
+ end
148
+ }
149
+ elsif @remove_prefix
150
+ self.instance_eval {
151
+ def format(tag,time,record)
152
+ if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
153
+ tag == @remove_prefix
154
+ tag = tag[@removed_length..-1] || @default_tag
155
+ end
156
+ tag + @f_separator + stringify_record(record)
157
+ end
158
+ }
159
+ else
160
+ self.instance_eval {
161
+ def format(tag,time,record)
162
+ tag + @f_separator + stringify_record(record)
163
+ end
164
+ }
165
+ end
166
+ else # without time, tag
167
+ if @add_newline
168
+ self.instance_eval {
169
+ def format(tag,time,record);
170
+ stringify_record(record) + "\n"
171
+ end
172
+ }
173
+ else
174
+ self.instance_eval {
175
+ def format(tag,time,record);
176
+ stringify_record(record)
177
+ end
178
+ }
179
+ end
180
+ end
181
+ end
182
+
183
+ def stringify_record(record)
184
+ record.to_json
185
+ end
186
+
187
+ def format(tag, time, record)
188
+ if tag == @remove_prefix or (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length)
189
+ tag = tag[@removed_length..-1] || @default_tag
190
+ end
191
+ time_str = if @output_include_time
192
+ @timef.format(time) + @f_separator
193
+ else
194
+ ''
195
+ end
196
+ tag_str = if @output_include_tag
197
+ tag + @f_separator
198
+ else
199
+ ''
200
+ end
201
+ time_str + tag_str + stringify_record(record) + "\n"
202
+ end
203
+
204
+ end
205
+
206
+ class Fluent::FileAlternativeOutput < Fluent::TimeSlicedOutput
207
+ Fluent::Plugin.register_output('file_alternative', self)
208
+
209
+ SUPPORTED_COMPRESS = {
210
+ :gz => :gz,
211
+ :gzip => :gz,
212
+ }
213
+
214
+ config_set_default :time_slice_format, '%Y%m%d' # %Y%m%d%H
215
+
216
+ config_param :path, :string # /path/pattern/to/hdfs/file can use %Y %m %d %H %M %S
217
+
218
+ config_param :compress, :default => nil do |val|
219
+ c = SUPPORTED_COMPRESS[val.to_sym]
220
+ unless c
221
+ raise ConfigError, "Unsupported compression algorithm '#{compress}'"
222
+ end
223
+ c
224
+ end
225
+
226
+ include FluentExt::PlainTextFormatterMixin
227
+
228
+ config_set_default :output_include_time, true
229
+ config_set_default :output_include_tag, true
230
+ config_set_default :output_data_type, 'json'
231
+ config_set_default :field_separator, "\t"
232
+ config_set_default :add_newline, true
233
+ config_set_default :remove_prefix, nil
234
+
235
+ def initialize
236
+ super
237
+ require 'time'
238
+ require 'zlib'
239
+ end
240
+
241
+ def configure(conf)
242
+ if conf['path']
243
+ if conf['path'].index('%S')
244
+ conf['time_slice_format'] = '%Y%m%d%H%M%S'
245
+ elsif conf['path'].index('%M')
246
+ conf['time_slice_format'] = '%Y%m%d%H%M'
247
+ elsif conf['path'].index('%H')
248
+ conf['time_slice_format'] = '%Y%m%d%H'
249
+ end
250
+ end
251
+ if pos = (conf['path'] || '').index('*')
252
+ @path_prefix = conf['path'][0,pos]
253
+ @path_suffix = conf['path'][pos+1..-1]
254
+ conf['buffer_path'] ||= "#{conf['path']}"
255
+ elsif (conf['path'] || '%Y') !~ /%Y|%m|%d|%H|%M|%S/
256
+ @path_prefix = conf['path'] + "."
257
+ @path_suffix = ".log"
258
+ conf['buffer_path'] ||= "#{conf['path']}.*"
259
+ elsif (conf['path'] || '') =~ /%Y|%m|%d|%H|%M|%S/
260
+ conf['buffer_path'] ||= conf['path'].gsub('%Y','yyyy').gsub('%m','mm').gsub('%d','dd').gsub('%H','HH').gsub('%M','MM').gsub('%S','SS')
261
+ end
262
+
263
+ super
264
+
265
+ unless @path.index('/') == 0
266
+ raise Fluent::ConfigError, "Path on filesystem MUST starts with '/', but '#{@path}'"
267
+ end
268
+ end
269
+
270
+ def start
271
+ super
272
+ # init
273
+ end
274
+
275
+ def shutdown
276
+ super
277
+ # destroy
278
+ end
279
+
280
+ def record_to_string(record)
281
+ record.to_json
282
+ end
283
+
284
+ def format(tag, time, record)
285
+ time_str = @timef.format(time)
286
+ time_str + @f_separator + tag + @f_separator + record_to_string(record) + @line_end
287
+ end
288
+
289
+ def path_format(chunk_key)
290
+ suffix = case @compress
291
+ when :gz
292
+ '.gz'
293
+ else
294
+ ''
295
+ end
296
+ if @path_prefix and @path_suffix
297
+ if @compress
298
+ i = 0
299
+ begin
300
+ path = "#{@path_prefix}#{chunk_key}_#{i}#{@path_suffix}#{suffix}"
301
+ i += 1
302
+ end while File.exist?(path)
303
+ path
304
+ else
305
+ "#{@path_prefix}#{chunk_key}_#{@path_suffix}#{suffix}"
306
+ end
307
+ else
308
+ if @compress
309
+ path_base = Time.strptime(chunk_key, @time_slice_format).strftime(@path)
310
+ path = path_base + suffix
311
+ if File.exist?(path)
312
+ i = 0
313
+ begin
314
+ path = "#{path_base}.#{i}#{suffix}"
315
+ i += 1
316
+ end while File.exist?(path)
317
+ end
318
+ path
319
+ else
320
+ Time.strptime(chunk_key, @time_slice_format).strftime(@path)
321
+ end
322
+ end
323
+ end
324
+
325
+ def write(chunk)
326
+ path = path_format(chunk.key)
327
+
328
+ begin
329
+ FileUtils.mkdir_p File.dirname(path)
330
+
331
+ case @compress
332
+ when :gz
333
+ Zlib::GzipWriter.open(path) {|f|
334
+ chunk.write_to(f)
335
+ }
336
+ else
337
+ File.open(path, "a") {|f|
338
+ chunk.write_to(f)
339
+ }
340
+ end
341
+ rescue
342
+ $log.error "failed to write data: path #{path}"
343
+ raise
344
+ end
345
+ path
346
+ end
347
+ end
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_file_alternative'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,120 @@
1
+ require 'helper'
2
+
3
+ class FileAlternativeOutputTest < Test::Unit::TestCase
4
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp"
5
+
6
+ CONFIG = %[
7
+ path #{TMP_DIR}/accesslog.%Y-%m-%d
8
+ compress gz
9
+ ]
10
+
11
+ def setup
12
+ Fluent::Test.setup
13
+ FileUtils.rm_rf(TMP_DIR)
14
+ FileUtils.mkdir_p(TMP_DIR)
15
+ end
16
+
17
+ def create_driver(conf = CONFIG, tag='test')
18
+ Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::FileAlternativeOutput, tag).configure(conf)
19
+ end
20
+
21
+ # config_param :output_data_type, :string, :default => 'json' # or 'attr:field' or 'attr:field1,field2,field3(...)'
22
+
23
+ # path
24
+ # output_include_time , output_include_tag , output_data_type
25
+ # add_newline , field_separator
26
+ # remove_prefix , :default_tag
27
+ def test_configure
28
+ # many many many tests should be written, for PlainTextFormatterMixin ...
29
+
30
+ d = create_driver %[
31
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
32
+ ]
33
+ assert_equal '%Y%m%d%H%M%S', d.instance.time_slice_format
34
+ assert_nil d.instance.compress
35
+ assert_equal true, d.instance.output_include_time
36
+ assert_equal true, d.instance.output_include_tag
37
+ assert_equal 'json', d.instance.output_data_type
38
+ assert_equal true, d.instance.add_newline
39
+ assert_equal "\t", d.instance.field_separator
40
+ assert_nil d.instance.remove_prefix
41
+ end
42
+
43
+ def test_format
44
+ d1 = create_driver
45
+
46
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
47
+ d1.emit({"a"=>1}, time)
48
+ d1.emit({"a"=>2}, time)
49
+ d1.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
50
+ d1.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
51
+ d1.run
52
+
53
+ d2 = create_driver %[
54
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
55
+ output_include_time false
56
+ output_include_tag false
57
+ output_data_type attr:message
58
+ ]
59
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
60
+ d2.emit({"message" => "abc-xyz-123", "other" => "zzz"}, time)
61
+ d2.emit({"message" => "123-456-789", "other" => "ppp"}, time)
62
+ d2.expect_format %[abc-xyz-123\n]
63
+ d2.expect_format %[123-456-789\n]
64
+ d2.run
65
+
66
+ d3 = create_driver %[
67
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
68
+ output_include_time false
69
+ output_include_tag false
70
+ output_data_type attr:server,level,log
71
+ field_separator COMMA
72
+ add_newline false
73
+ ]
74
+ assert_equal ',', d3.instance.f_separator
75
+
76
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
77
+ d3.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
78
+ d3.emit({"server" => "app01", "level" => "info", "log" => "Send response\n"}, time)
79
+ d3.expect_format %[www01,warn,Exception\n]
80
+ d3.expect_format %[app01,info,Send response\n]
81
+ d3.run
82
+ end
83
+
84
+ def test_write
85
+ d2 = create_driver %[
86
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
87
+ output_include_time false
88
+ output_include_tag false
89
+ output_data_type attr:message
90
+ utc
91
+ ]
92
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
93
+ d2.emit({"message" => "abc-xyz-123", "other" => "zzz"}, time)
94
+ d2.emit({"message" => "123-456-789", "other" => "ppp"}, time)
95
+ d2.expect_format %[abc-xyz-123\n]
96
+ d2.expect_format %[123-456-789\n]
97
+ path = d2.run
98
+ assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15", path[0]
99
+
100
+ d3 = create_driver %[
101
+ path #{TMP_DIR}/accesslog.%Y-%m-%d-%H-%M-%S
102
+ compress gzip
103
+ output_include_time false
104
+ output_include_tag false
105
+ output_data_type attr:server,level,log
106
+ field_separator COMMA
107
+ add_newline false
108
+ utc
109
+ ]
110
+ assert_equal ',', d3.instance.f_separator
111
+
112
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
113
+ d3.emit({"server" => "www01", "level" => "warn", "log" => "Exception\n"}, time)
114
+ d3.emit({"server" => "app01", "level" => "info", "log" => "Send response\n"}, time)
115
+ d3.expect_format %[www01,warn,Exception\n]
116
+ d3.expect_format %[app01,info,Send response\n]
117
+ path = d3.run
118
+ assert_equal "#{TMP_DIR}/accesslog.2011-01-02-13-14-15.gz", path[0]
119
+ end
120
+ end
@@ -0,0 +1,81 @@
1
+ require 'fluent/test'
2
+ require 'fileutils'
3
+ require 'time'
4
+
5
+ class FileOutputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ FileUtils.rm_rf(TMP_DIR)
9
+ FileUtils.mkdir_p(TMP_DIR)
10
+ end
11
+
12
+ TMP_DIR = File.dirname(__FILE__) + "/../tmp"
13
+
14
+ CONFIG = %[
15
+ path #{TMP_DIR}/out_file_test
16
+ compress gz
17
+ utc
18
+ ]
19
+
20
+ def create_driver(conf = CONFIG)
21
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
22
+ end
23
+
24
+ def test_configure
25
+ d = create_driver %[
26
+ path test_path
27
+ compress gz
28
+ ]
29
+ assert_equal 'test_path', d.instance.path
30
+ assert_equal :gz, d.instance.compress
31
+ end
32
+
33
+ def test_format
34
+ d = create_driver
35
+
36
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
37
+ d.emit({"a"=>1}, time)
38
+ d.emit({"a"=>2}, time)
39
+
40
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
41
+ d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
42
+
43
+ d.run
44
+ end
45
+
46
+ def test_write
47
+ d = create_driver
48
+
49
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
50
+ d.emit({"a"=>1}, time)
51
+ d.emit({"a"=>2}, time)
52
+
53
+ # FileOutput#write returns path
54
+ path = d.run
55
+ expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
56
+ assert_equal expect_path, path
57
+
58
+ data = Zlib::GzipReader.open(expect_path) {|f| f.read }
59
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
60
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
61
+ data
62
+ end
63
+
64
+ def test_write_path_increment
65
+ d = create_driver
66
+
67
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
68
+ d.emit({"a"=>1}, time)
69
+ d.emit({"a"=>2}, time)
70
+
71
+ # FileOutput#write returns path
72
+ path = d.run
73
+ assert_equal "#{TMP_DIR}/out_file_test._0.log.gz", path
74
+
75
+ path = d.run
76
+ assert_equal "#{TMP_DIR}/out_file_test._1.log.gz", path
77
+
78
+ path = d.run
79
+ assert_equal "#{TMP_DIR}/out_file_test._2.log.gz", path
80
+ end
81
+ end
metadata ADDED
@@ -0,0 +1,80 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-file-alternative
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - TAGOMORI Satoshi
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-03-16 00:00:00.000000000Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: &2168521620 !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: *2168521620
25
+ - !ruby/object:Gem::Dependency
26
+ name: fluentd
27
+ requirement: &2168521180 !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: *2168521180
36
+ description: alternative implementation of out_file, with various configurations
37
+ email:
38
+ - tagomoris@gmail.com
39
+ executables: []
40
+ extensions: []
41
+ extra_rdoc_files: []
42
+ files:
43
+ - .gitignore
44
+ - Gemfile
45
+ - LICENSE.txt
46
+ - README.md
47
+ - Rakefile
48
+ - fluent-plugin-file-alternative.gemspec
49
+ - lib/fluent/plugin/out_file_alternative.rb
50
+ - test/helper.rb
51
+ - test/plugin/test_out_file_alternative.rb
52
+ - test/plugin/test_out_file_original.rb
53
+ homepage: https://github.com/tagomoris/fluent-plugin-file-alternative
54
+ licenses: []
55
+ post_install_message:
56
+ rdoc_options: []
57
+ require_paths:
58
+ - lib
59
+ required_ruby_version: !ruby/object:Gem::Requirement
60
+ none: false
61
+ requirements:
62
+ - - ! '>='
63
+ - !ruby/object:Gem::Version
64
+ version: '0'
65
+ required_rubygems_version: !ruby/object:Gem::Requirement
66
+ none: false
67
+ requirements:
68
+ - - ! '>='
69
+ - !ruby/object:Gem::Version
70
+ version: '0'
71
+ requirements: []
72
+ rubyforge_project:
73
+ rubygems_version: 1.8.6
74
+ signing_key:
75
+ specification_version: 3
76
+ summary: alternative implementation of out_file
77
+ test_files:
78
+ - test/helper.rb
79
+ - test/plugin/test_out_file_alternative.rb
80
+ - test/plugin/test_out_file_original.rb