fluent-plugin-fork 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/.travis.yml +12 -0
- data/Gemfile +3 -0
- data/LICENSE +20 -0
- data/README.md +88 -0
- data/Rakefile +8 -0
- data/fluent-plugin-fork.gemspec +22 -0
- data/lib/fluent/plugin/out_fork.rb +64 -0
- data/spec/plugin/out_fork_spec.rb +117 -0
- data/spec/spec_helper.rb +10 -0
- metadata +112 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9a6046f60ea7ea3def073da545f25629badae3e2
|
4
|
+
data.tar.gz: 89c121b947e6e4ecb7c578eb00890f68cc36f8b1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 68a3e769a9fb09e3086026d3ce4ccefd19f488e5949285a8810c1268945b0695a8fd4009e6e10506bd6ae9821275baad6588101e142e6c332bc591ee82fbcb25
|
7
|
+
data.tar.gz: 9dee3e9e0ba50b0d03576044e44057780d97d3935d010013b4bb06a069c14a3d0347145f08dbd6d9775a1119c3532e1e121faeed17955c0dcdcd31a2b11e9534
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2014 Daisuke Taniwaki
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
[![Build Status](https://secure.travis-ci.org/dtaniwaki/fluent-plugin-fork.png?branch=master)](http://travis-ci.org/dtaniwaki/fluent-plugin-fork) [![Coverage Status](https://coveralls.io/repos/dtaniwaki/fluent-plugin-fork/badge.png?branch=master)](https://coveralls.io/r/dtaniwaki/fluent-plugin-fork?branch=master)
|
2
|
+
|
3
|
+
# fluent-plugin-fork
|
4
|
+
|
5
|
+
Fork output by separating values for fluentd
|
6
|
+
|
7
|
+
# Installation
|
8
|
+
|
9
|
+
### td-agent(Linux)
|
10
|
+
|
11
|
+
```
|
12
|
+
/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-fork
|
13
|
+
```
|
14
|
+
|
15
|
+
### td-agent(Mac)
|
16
|
+
|
17
|
+
```
|
18
|
+
sudo /usr/local/Cellar/td-agent/1.1.XX/bin/fluent-gem install fluent-plugin-fork
|
19
|
+
```
|
20
|
+
|
21
|
+
### fluentd only
|
22
|
+
|
23
|
+
```
|
24
|
+
gem install fluent-plugin-fork
|
25
|
+
```
|
26
|
+
|
27
|
+
# Configuration
|
28
|
+
|
29
|
+
```
|
30
|
+
output_tag tag_to_output
|
31
|
+
output_key key_to_output
|
32
|
+
separator ,
|
33
|
+
fork_key key_to_fork
|
34
|
+
max_size 15
|
35
|
+
max_fallback log
|
36
|
+
no_unique true
|
37
|
+
```
|
38
|
+
|
39
|
+
### output_tag
|
40
|
+
|
41
|
+
Tag to output forked values
|
42
|
+
|
43
|
+
### output_key
|
44
|
+
|
45
|
+
Key name to output forked values
|
46
|
+
|
47
|
+
### fork_key
|
48
|
+
|
49
|
+
Key name to fork
|
50
|
+
|
51
|
+
### separator (Optional)
|
52
|
+
|
53
|
+
Separator to separate the values
|
54
|
+
|
55
|
+
Default: `,`
|
56
|
+
|
57
|
+
### max_size (Optional)
|
58
|
+
|
59
|
+
Max size of forked values.
|
60
|
+
|
61
|
+
Default: `nil`
|
62
|
+
|
63
|
+
### max_fallback (Optional)
|
64
|
+
|
65
|
+
Strategy when the size of values exceeds max_size. Only effective when you set max_size.
|
66
|
+
|
67
|
+
`log` to log the record
|
68
|
+
`drop` to drop exceeded values
|
69
|
+
|
70
|
+
Default: `log`
|
71
|
+
|
72
|
+
### no_unique (Optional)
|
73
|
+
|
74
|
+
Flag to emit redundant values.
|
75
|
+
|
76
|
+
Default: `false`
|
77
|
+
|
78
|
+
# Contributing
|
79
|
+
|
80
|
+
1. Fork it
|
81
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
82
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
83
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
84
|
+
5. Create new [Pull Request](../../pull/new/master)
|
85
|
+
|
86
|
+
# Copyright
|
87
|
+
|
88
|
+
Copyright (c) 2014 Daisuke Taniwaki. See [LICENSE](LICENSE) for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
$:.push File.expand_path('../lib', __FILE__)
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-fork"
|
5
|
+
gem.version = "0.1.0"
|
6
|
+
gem.authors = ["Daisuke Taniwaki"]
|
7
|
+
gem.email = "daisuketaniwaki@gmail.com"
|
8
|
+
gem.homepage = "https://github.com/dtaniwaki/fluent-plugin-fork"
|
9
|
+
gem.description = "Fork output by separating values for fluentd"
|
10
|
+
gem.summary = "Fork output by separating values for fluentd"
|
11
|
+
gem.licenses = ["MIT"]
|
12
|
+
|
13
|
+
gem.files = `git ls-files`.split("\n")
|
14
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
15
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
16
|
+
gem.require_paths = ['lib']
|
17
|
+
|
18
|
+
gem.add_runtime_dependency "fluentd"
|
19
|
+
gem.add_development_dependency "rake"
|
20
|
+
gem.add_development_dependency "rspec"
|
21
|
+
gem.add_development_dependency "coveralls"
|
22
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
module Fluent
|
2
|
+
class ForkOutput < Output
|
3
|
+
class MaxForkSizeError < StandardError; end
|
4
|
+
|
5
|
+
Fluent::Plugin.register_output('fork', self)
|
6
|
+
|
7
|
+
unless method_defined?(:log)
|
8
|
+
define_method(:log) { $log }
|
9
|
+
end
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
config_param :output_tag, :string
|
16
|
+
config_param :output_key, :string
|
17
|
+
config_param :fork_key, :string
|
18
|
+
config_param :separator, :string, default: ','
|
19
|
+
config_param :max_size, :integer, default: nil
|
20
|
+
config_param :max_fallback, :string, default: 'log'
|
21
|
+
config_param :no_unique, :bool, default: false
|
22
|
+
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
|
26
|
+
fallbacks = %w(drop log)
|
27
|
+
raise Fluent::ConfigError, "max_fallback must be one of #{fallbacks.inspect}" unless fallbacks.include?(@max_fallback)
|
28
|
+
end
|
29
|
+
|
30
|
+
def emit(tag, es, chain)
|
31
|
+
es.each do |time, record|
|
32
|
+
org_value = record[@fork_key]
|
33
|
+
if org_value.nil?
|
34
|
+
log.trace "#{tag} - #{time}: skip to fork #{@fork_key}=#{org_value}"
|
35
|
+
next
|
36
|
+
end
|
37
|
+
log.trace "#{tag} - #{time}: try to fork #{@fork_key}=#{org_value}"
|
38
|
+
|
39
|
+
values = org_value.split(@separator)
|
40
|
+
|
41
|
+
values = values.uniq unless @no_unique
|
42
|
+
|
43
|
+
if @max_size && @max_size < values.size
|
44
|
+
case @max_fallback
|
45
|
+
when 'drop'
|
46
|
+
log.warn "#{tag} - #{time}: Drop too many forked values (max=#{@max_size}) : #{org_value}"
|
47
|
+
values = values.take(@max_size)
|
48
|
+
when 'log'
|
49
|
+
log.info "#{tag} - #{time}: Too many forked values (max=#{@max_size}) : #{org_value}"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
values.reject{ |value| value.to_s == '' }.each do |value|
|
54
|
+
log.trace "#{tag} - #{time}: reemit #{@output_key}=#{value} for #{@output_tag}"
|
55
|
+
Engine.emit(@output_tag, time, record.reject{ |k, v| k == @fork_key }.merge(@output_key => value))
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
chain.next
|
60
|
+
rescue => e
|
61
|
+
log.error "#{e.message}: #{e.backtrace.join(', ')}"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Fluent::ForkOutput do
|
4
|
+
let(:tag) { "test.fork" }
|
5
|
+
let(:required_params) { {output_tag: 'ot', output_key: 'ok', fork_key: 'sk'} }
|
6
|
+
let(:params) { required_params }
|
7
|
+
let(:config) { params.map{ |k, v| "#{k} #{v}" }.join("\n") }
|
8
|
+
subject { Fluent::Test::OutputTestDriver.new(Fluent::ForkOutput, tag).configure(config) }
|
9
|
+
|
10
|
+
describe "#configure" do
|
11
|
+
let(:params) { required_params.merge(separator: '-', max_size: 5, max_fallback: 'drop', no_unique: true) }
|
12
|
+
it "does not raise any error" do
|
13
|
+
expect{ subject }.not_to raise_error
|
14
|
+
expect(subject.instance.output_tag).to eq('ot')
|
15
|
+
expect(subject.instance.output_key).to eq('ok')
|
16
|
+
expect(subject.instance.fork_key).to eq('sk')
|
17
|
+
expect(subject.instance.separator).to eq('-')
|
18
|
+
expect(subject.instance.no_unique).to eq(true)
|
19
|
+
expect(subject.instance.max_size).to eq(5)
|
20
|
+
expect(subject.instance.max_fallback).to eq('drop')
|
21
|
+
end
|
22
|
+
context "without optional params" do
|
23
|
+
let(:params) { required_params }
|
24
|
+
it "use default values" do
|
25
|
+
expect{ subject }.not_to raise_error
|
26
|
+
expect(subject.instance.output_tag).to eq('ot')
|
27
|
+
expect(subject.instance.output_key).to eq('ok')
|
28
|
+
expect(subject.instance.fork_key).to eq('sk')
|
29
|
+
expect(subject.instance.separator).to eq(',')
|
30
|
+
expect(subject.instance.no_unique).to eq(false)
|
31
|
+
expect(subject.instance.max_size).to eq(nil)
|
32
|
+
expect(subject.instance.max_fallback).to eq('log')
|
33
|
+
end
|
34
|
+
end
|
35
|
+
context "with invalid fallback" do
|
36
|
+
let(:params) { required_params.merge(max_fallback: 'invalid') }
|
37
|
+
it "raise an error" do
|
38
|
+
expect{ subject }.to raise_error(Fluent::ConfigError)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
[:output_tag, :output_key, :fork_key].each do |rk|
|
42
|
+
context "no #{rk}" do
|
43
|
+
let(:params) { required_params.reject{ |k, v| k == rk } }
|
44
|
+
it "raises Fluent::ConfigError" do
|
45
|
+
expect{ subject }.to raise_error(Fluent::ConfigError)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe "#run" do
|
52
|
+
let(:time) { Time.now.to_i }
|
53
|
+
it "forks" do
|
54
|
+
subject.run { subject.emit({"sk" => "2,3,4,5"}, time) }
|
55
|
+
expect(subject.emits.size).to eq(4)
|
56
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
|
57
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
|
58
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
|
59
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "5"}])
|
60
|
+
end
|
61
|
+
it "forks uniquely" do
|
62
|
+
subject.run { subject.emit({"sk" => "2,3,4,3"}, time) }
|
63
|
+
expect(subject.emits.size).to eq(3)
|
64
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
|
65
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
|
66
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
|
67
|
+
end
|
68
|
+
it "forks with other params" do
|
69
|
+
subject.run { subject.emit({"sk" => "2,3,4,3", "o1" => 1, "o2" => 2}, time) }
|
70
|
+
expect(subject.emits.size).to eq(3)
|
71
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "2", "o1" => 1, "o2" => 2}])
|
72
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "3", "o1" => 1, "o2" => 2}])
|
73
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "4", "o1" => 1, "o2" => 2}])
|
74
|
+
end
|
75
|
+
context "with no_unique option" do
|
76
|
+
let(:params) { required_params.merge(no_unique: true) }
|
77
|
+
it "forks for redundant values" do
|
78
|
+
subject.run { subject.emit({"sk" => "2,3,4,3"}, time) }
|
79
|
+
expect(subject.emits.size).to eq(4)
|
80
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
|
81
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
|
82
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
|
83
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
|
84
|
+
end
|
85
|
+
end
|
86
|
+
context "with separator option" do
|
87
|
+
let(:params) { required_params.merge(separator: '-') }
|
88
|
+
it "forks by separating with '-'" do
|
89
|
+
subject.run { subject.emit({"sk" => "2-3-4-5"}, time) }
|
90
|
+
expect(subject.emits.size).to eq(4)
|
91
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
|
92
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
|
93
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
|
94
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "5"}])
|
95
|
+
end
|
96
|
+
end
|
97
|
+
context "with max_size and max_fallback options" do
|
98
|
+
describe "log" do
|
99
|
+
let(:params) { required_params.merge(max_size: 3, max_fallback: 'log') }
|
100
|
+
it "writes a log" do
|
101
|
+
expect(subject.instance.log).to receive(:info).with(/Too many forked values/)
|
102
|
+
subject.run { subject.emit({"sk" => "2,3,4,5"}, time) }
|
103
|
+
end
|
104
|
+
end
|
105
|
+
describe "drop" do
|
106
|
+
let(:params) { required_params.merge(max_size: 3, max_fallback: 'drop') }
|
107
|
+
it "drops exceeded values" do
|
108
|
+
subject.run { subject.emit({"sk" => "2,3,4,5"}, time) }
|
109
|
+
expect(subject.emits.size).to eq(3)
|
110
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "2"}])
|
111
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "3"}])
|
112
|
+
expect(subject.emits).to include(["ot", time, {"ok" => "4"}])
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-fork
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Daisuke Taniwaki
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-05-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: coveralls
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: Fork output by separating values for fluentd
|
70
|
+
email: daisuketaniwaki@gmail.com
|
71
|
+
executables: []
|
72
|
+
extensions: []
|
73
|
+
extra_rdoc_files: []
|
74
|
+
files:
|
75
|
+
- ".gitignore"
|
76
|
+
- ".travis.yml"
|
77
|
+
- Gemfile
|
78
|
+
- LICENSE
|
79
|
+
- README.md
|
80
|
+
- Rakefile
|
81
|
+
- fluent-plugin-fork.gemspec
|
82
|
+
- lib/fluent/plugin/out_fork.rb
|
83
|
+
- spec/plugin/out_fork_spec.rb
|
84
|
+
- spec/spec_helper.rb
|
85
|
+
homepage: https://github.com/dtaniwaki/fluent-plugin-fork
|
86
|
+
licenses:
|
87
|
+
- MIT
|
88
|
+
metadata: {}
|
89
|
+
post_install_message:
|
90
|
+
rdoc_options: []
|
91
|
+
require_paths:
|
92
|
+
- lib
|
93
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ">="
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0'
|
98
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
requirements: []
|
104
|
+
rubyforge_project:
|
105
|
+
rubygems_version: 2.0.14
|
106
|
+
signing_key:
|
107
|
+
specification_version: 4
|
108
|
+
summary: Fork output by separating values for fluentd
|
109
|
+
test_files:
|
110
|
+
- spec/plugin/out_fork_spec.rb
|
111
|
+
- spec/spec_helper.rb
|
112
|
+
has_rdoc:
|