fluent-plugin-tagfile 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,41 @@
1
+ #fluent-plugin-tagfile
2
+
3
+ This is similar to the `file` build-in output plugin, but `tag_file` decides output directory using tag of events.
4
+
5
+ ##Installation
6
+
7
+ Puts `out_tag_file.rb` to plugin directory.
8
+
9
+ ```shell
10
+ % cp out_tag_file.rb path/to/fluent/plugin
11
+ ```
12
+
13
+ ##Example
14
+
15
+ ```conf
16
+ #/etc/fluentd/fluent.conf
17
+ <match prefix.**>
18
+ type tag_file
19
+
20
+ path /var/log/fluent
21
+ compress gzip
22
+
23
+ time_slice_format %Y/%m/%d/%H/%M
24
+ flush_interval 60s
25
+ </match>
26
+ ```
27
+
28
+ Fluent with such conf file behaves as follows.
29
+ Suppose that tag is `prefix.foo.bar` and time is `2012/02/01 18:46`.
30
+
31
+ 1. Look for all events whose tag starts with `prefix.`.
32
+
33
+ 2. Create buffer file in `path` directory such as `/var/log/fluent/buffer.xxxxx`.
34
+
35
+ 3. In every minutes, fluent tries to flush the buffer, then `/var/log/fluent/foo/bar/2012/02/01/18/46/N.log.gz` is created. N is a unique number in the directory.
36
+
37
+ If `time_slice_format` includes `/` like this example, it means the directory hierarchy.
38
+
39
+ ##See also
40
+
41
+ http://fluentd.org/doc/plugin.html#file
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
@@ -0,0 +1,18 @@
1
+ # -*- coding:utf-8 -*-
2
+
3
+ Gem::Specification.new do |gem|
4
+ gem.authors = ["taka84u9"]
5
+ gem.email = ["taka84u9@gmail.com"]
6
+ gem.description = %q{Fluent output plugin to handle output directory by source host using events tag.}
7
+ gem.summary = %q{Fluent output plugin to handle output directory by source host using events tag.}
8
+ gem.homepage = "https://github.com/taka84u9/fluent-plugin-tagfile"
9
+
10
+ gem.files = `git ls-files`.split($\)
11
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
12
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
13
+ gem.name = "fluent-plugin-tagfile"
14
+ gem.require_paths = ["lib"]
15
+ gem.version = "0.0.1"
16
+ gem.add_development_dependency "fluentd"
17
+ gem.add_runtime_dependency "fluentd"
18
+ end
@@ -0,0 +1,59 @@
1
+ module Fluent
2
+ class TagFileOutput < FileOutput
3
+ Plugin.register_output('tag_file', self)
4
+
5
+ def configure(conf)
6
+ conf['buffer_path'] ||= File.join(conf['path'], 'buffer')
7
+
8
+ super
9
+ end
10
+
11
+ def format(tag, time, record)
12
+ tag_elems = tag.split('.')
13
+ tag_elems.shift # remove PREFIX
14
+ dir = File.join(@path, *tag_elems)
15
+
16
+ # @timef is assigned in FileOutput.configure
17
+ time_str = @timef.format(time)
18
+
19
+ [dir, "#{time_str}\t#{Yajl.dump(record)}\n"].to_msgpack
20
+ end
21
+
22
+ def write(chunk)
23
+ case @compress
24
+ when nil
25
+ suffix = ''
26
+ when :gz
27
+ suffix = '.gz'
28
+ end
29
+
30
+ hash = {}
31
+ chunk.msgpack_each do |(dir, data)|
32
+ sym = dir.to_sym
33
+ if hash.include?(sym)
34
+ hash[sym] += data
35
+ else
36
+ hash[sym] = data
37
+ end
38
+ end
39
+
40
+ hash.each do |dir, data|
41
+ dir = File.join(dir.to_s, chunk.key)
42
+
43
+ i = 0
44
+ begin
45
+ path = File.join(dir, "#{i}.log#{suffix}")
46
+ i += 1
47
+ end while File.exist?(path)
48
+ FileUtils.mkdir_p dir
49
+
50
+ case @compress
51
+ when nil
52
+ File.open(path, 'a') {|f| f.write(data) }
53
+ when :gz
54
+ Zlib::GzipWriter.open(path) {|f| f.write(data) }
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
metadata ADDED
@@ -0,0 +1,73 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-tagfile
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - taka84u9
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2012-06-05 00:00:00.000000000Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: &70291876566980 !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: *70291876566980
25
+ - !ruby/object:Gem::Dependency
26
+ name: fluentd
27
+ requirement: &70291876566540 !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: *70291876566540
36
+ description: Fluent output plugin to handle output directory by source host using
37
+ events tag.
38
+ email:
39
+ - taka84u9@gmail.com
40
+ executables: []
41
+ extensions: []
42
+ extra_rdoc_files: []
43
+ files:
44
+ - README.md
45
+ - Rakefile
46
+ - fluent-plugin-tagfile.gemspec
47
+ - lib/fluent/plugin/out_tag_file.rb
48
+ homepage: https://github.com/taka84u9/fluent-plugin-tagfile
49
+ licenses: []
50
+ post_install_message:
51
+ rdoc_options: []
52
+ require_paths:
53
+ - lib
54
+ required_ruby_version: !ruby/object:Gem::Requirement
55
+ none: false
56
+ requirements:
57
+ - - ! '>='
58
+ - !ruby/object:Gem::Version
59
+ version: '0'
60
+ required_rubygems_version: !ruby/object:Gem::Requirement
61
+ none: false
62
+ requirements:
63
+ - - ! '>='
64
+ - !ruby/object:Gem::Version
65
+ version: '0'
66
+ requirements: []
67
+ rubyforge_project:
68
+ rubygems_version: 1.8.10
69
+ signing_key:
70
+ specification_version: 3
71
+ summary: Fluent output plugin to handle output directory by source host using events
72
+ tag.
73
+ test_files: []