fluent-plugin-tagfile 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +41 -0
- data/Rakefile +2 -0
- data/fluent-plugin-tagfile.gemspec +18 -0
- data/lib/fluent/plugin/out_tag_file.rb +59 -0
- metadata +73 -0
data/README.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
#fluent-plugin-tagfile
|
2
|
+
|
3
|
+
This is similar to the `file` build-in output plugin, but `tag_file` decides output directory using tag of events.
|
4
|
+
|
5
|
+
##Installation
|
6
|
+
|
7
|
+
Puts `out_tag_file.rb` to plugin directory.
|
8
|
+
|
9
|
+
```shell
|
10
|
+
% cp out_tag_file.rb path/to/fluent/plugin
|
11
|
+
```
|
12
|
+
|
13
|
+
##Example
|
14
|
+
|
15
|
+
```conf
|
16
|
+
#/etc/fluentd/fluent.conf
|
17
|
+
<match prefix.**>
|
18
|
+
type tag_file
|
19
|
+
|
20
|
+
path /var/log/fluent
|
21
|
+
compress gzip
|
22
|
+
|
23
|
+
time_slice_format %Y/%m/%d/%H/%M
|
24
|
+
flush_interval 60s
|
25
|
+
</match>
|
26
|
+
```
|
27
|
+
|
28
|
+
Fluent with such conf file behaves as follows.
|
29
|
+
Suppose that tag is `prefix.foo.bar` and time is `2012/02/01 18:46`.
|
30
|
+
|
31
|
+
1. Look for all events whose tag starts with `prefix.`.
|
32
|
+
|
33
|
+
2. Create buffer file in `path` directory such as `/var/log/fluent/buffer.xxxxx`.
|
34
|
+
|
35
|
+
3. In every minutes, fluent tries to flush the buffer, then `/var/log/fluent/foo/bar/2012/02/01/18/46/N.log.gz` is created. N is a unique number in the directory.
|
36
|
+
|
37
|
+
If `time_slice_format` includes `/` like this example, it means the directory hierarchy.
|
38
|
+
|
39
|
+
##See also
|
40
|
+
|
41
|
+
http://fluentd.org/doc/plugin.html#file
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# -*- coding:utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.authors = ["taka84u9"]
|
5
|
+
gem.email = ["taka84u9@gmail.com"]
|
6
|
+
gem.description = %q{Fluent output plugin to handle output directory by source host using events tag.}
|
7
|
+
gem.summary = %q{Fluent output plugin to handle output directory by source host using events tag.}
|
8
|
+
gem.homepage = "https://github.com/taka84u9/fluent-plugin-tagfile"
|
9
|
+
|
10
|
+
gem.files = `git ls-files`.split($\)
|
11
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
12
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
|
+
gem.name = "fluent-plugin-tagfile"
|
14
|
+
gem.require_paths = ["lib"]
|
15
|
+
gem.version = "0.0.1"
|
16
|
+
gem.add_development_dependency "fluentd"
|
17
|
+
gem.add_runtime_dependency "fluentd"
|
18
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module Fluent
|
2
|
+
class TagFileOutput < FileOutput
|
3
|
+
Plugin.register_output('tag_file', self)
|
4
|
+
|
5
|
+
def configure(conf)
|
6
|
+
conf['buffer_path'] ||= File.join(conf['path'], 'buffer')
|
7
|
+
|
8
|
+
super
|
9
|
+
end
|
10
|
+
|
11
|
+
def format(tag, time, record)
|
12
|
+
tag_elems = tag.split('.')
|
13
|
+
tag_elems.shift # remove PREFIX
|
14
|
+
dir = File.join(@path, *tag_elems)
|
15
|
+
|
16
|
+
# @timef is assigned in FileOutput.configure
|
17
|
+
time_str = @timef.format(time)
|
18
|
+
|
19
|
+
[dir, "#{time_str}\t#{Yajl.dump(record)}\n"].to_msgpack
|
20
|
+
end
|
21
|
+
|
22
|
+
def write(chunk)
|
23
|
+
case @compress
|
24
|
+
when nil
|
25
|
+
suffix = ''
|
26
|
+
when :gz
|
27
|
+
suffix = '.gz'
|
28
|
+
end
|
29
|
+
|
30
|
+
hash = {}
|
31
|
+
chunk.msgpack_each do |(dir, data)|
|
32
|
+
sym = dir.to_sym
|
33
|
+
if hash.include?(sym)
|
34
|
+
hash[sym] += data
|
35
|
+
else
|
36
|
+
hash[sym] = data
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
hash.each do |dir, data|
|
41
|
+
dir = File.join(dir.to_s, chunk.key)
|
42
|
+
|
43
|
+
i = 0
|
44
|
+
begin
|
45
|
+
path = File.join(dir, "#{i}.log#{suffix}")
|
46
|
+
i += 1
|
47
|
+
end while File.exist?(path)
|
48
|
+
FileUtils.mkdir_p dir
|
49
|
+
|
50
|
+
case @compress
|
51
|
+
when nil
|
52
|
+
File.open(path, 'a') {|f| f.write(data) }
|
53
|
+
when :gz
|
54
|
+
Zlib::GzipWriter.open(path) {|f| f.write(data) }
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
metadata
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-tagfile
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- taka84u9
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-06-05 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: &70291876566980 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *70291876566980
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: fluentd
|
27
|
+
requirement: &70291876566540 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *70291876566540
|
36
|
+
description: Fluent output plugin to handle output directory by source host using
|
37
|
+
events tag.
|
38
|
+
email:
|
39
|
+
- taka84u9@gmail.com
|
40
|
+
executables: []
|
41
|
+
extensions: []
|
42
|
+
extra_rdoc_files: []
|
43
|
+
files:
|
44
|
+
- README.md
|
45
|
+
- Rakefile
|
46
|
+
- fluent-plugin-tagfile.gemspec
|
47
|
+
- lib/fluent/plugin/out_tag_file.rb
|
48
|
+
homepage: https://github.com/taka84u9/fluent-plugin-tagfile
|
49
|
+
licenses: []
|
50
|
+
post_install_message:
|
51
|
+
rdoc_options: []
|
52
|
+
require_paths:
|
53
|
+
- lib
|
54
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
55
|
+
none: false
|
56
|
+
requirements:
|
57
|
+
- - ! '>='
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '0'
|
60
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ! '>='
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '0'
|
66
|
+
requirements: []
|
67
|
+
rubyforge_project:
|
68
|
+
rubygems_version: 1.8.10
|
69
|
+
signing_key:
|
70
|
+
specification_version: 3
|
71
|
+
summary: Fluent output plugin to handle output directory by source host using events
|
72
|
+
tag.
|
73
|
+
test_files: []
|