fluent-plugin-hash-forward 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +14 -0
- data/Gemfile +3 -0
- data/README.md +60 -0
- data/Rakefile +15 -0
- data/fluent-plugin-hash-forward.gemspec +27 -0
- data/lib/fluent/plugin/out_hash_forward.rb +79 -0
- data/spec/out_hash_forward_spec.rb +114 -0
- data/spec/spec_helper.rb +13 -0
- metadata +142 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 34a4f0ca25be43b69b64a1338daabc8372a2008b
|
4
|
+
data.tar.gz: be4337ab0434d3fe45be14de13dc381580d8d49d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e037f72f0fa5fbc9980e11a7a9ec94a336488b05dc45eeb69589182fa404a5c02921ae210277b0288e20eaf60002102a7e18d1051834830851a7b2af729ac0a4
|
7
|
+
data.tar.gz: 7a20b3102b6a98e193608b2f21aa45b0ebd5b5dfef0222a39ef8ad50f6a01ad4336a5ad4d63539ae96d6b601f47398969ea540dc471a69026ee6029131ac13e1
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
# fluent-plugin-hash-forward
|
2
|
+
|
3
|
+
Fluentd plugin to keep forwarding messages of a specific tag pattern to a specific node
|
4
|
+
|
5
|
+
- Forward some servers
|
6
|
+
- Same tag messages forward to the same server
|
7
|
+
- Using Murmurhash
|
8
|
+
|
9
|
+
## Configuration
|
10
|
+
|
11
|
+
Example:
|
12
|
+
|
13
|
+
<match pattern>
|
14
|
+
type hash_forward
|
15
|
+
flush_interval 1s
|
16
|
+
|
17
|
+
<server>
|
18
|
+
host 192.168.1.3
|
19
|
+
port 24224
|
20
|
+
</server>
|
21
|
+
<server>
|
22
|
+
host 192.168.1.4
|
23
|
+
port 24224
|
24
|
+
</server>
|
25
|
+
|
26
|
+
<secondary>
|
27
|
+
type file
|
28
|
+
path /var/log/fluent/forward-failed
|
29
|
+
</secondary>
|
30
|
+
</match>
|
31
|
+
|
32
|
+
## Parameters
|
33
|
+
|
34
|
+
Basically same with out\_forward plugin. See [http://docs.fluentd.org/articles/out_forward](http://docs.fluentd.org/ja/articles/out_forward).
|
35
|
+
|
36
|
+
Following parameters are additionally available:
|
37
|
+
|
38
|
+
* hash\_key
|
39
|
+
|
40
|
+
Specify a placeholder string to be used as a key for hashing. See Placeholders section for more details. Default uses `${tag}`as a hash key.
|
41
|
+
|
42
|
+
### Placeholders
|
43
|
+
|
44
|
+
You can use following placeholders:
|
45
|
+
|
46
|
+
* ${tag} input tag
|
47
|
+
* ${tags} input tag splitted by '.'
|
48
|
+
|
49
|
+
It is also possible to write a ruby code in placeholders, so you may write some codes as
|
50
|
+
|
51
|
+
* ${tags[0]}
|
52
|
+
* ${tags.last}
|
53
|
+
|
54
|
+
## Copyright
|
55
|
+
|
56
|
+
* Copyright
|
57
|
+
* Copyright (c) 2012- Ryosuke IWANAGA (riywo)
|
58
|
+
* Copyright (c) 2013- Naotoshi SEO (sonots)
|
59
|
+
* License
|
60
|
+
* Apache License, Version 2.0
|
data/Rakefile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
|
4
|
+
require 'rspec/core'
|
5
|
+
require 'rspec/core/rake_task'
|
6
|
+
RSpec::Core::RakeTask.new(:spec) do |spec|
|
7
|
+
spec.pattern = FileList['spec/**/*_spec.rb']
|
8
|
+
end
|
9
|
+
task :default => :spec
|
10
|
+
|
11
|
+
desc 'Open an irb session preloaded with the gem library'
|
12
|
+
task :console do
|
13
|
+
sh 'irb -rubygems -I lib'
|
14
|
+
end
|
15
|
+
task :c => :console
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = "fluent-plugin-hash-forward"
|
6
|
+
s.version = "0.0.1"
|
7
|
+
s.authors = ["Ryosuke IWANAGA", "Naotoshi SEO"]
|
8
|
+
s.email = ["riywo.jp@gmail.com", "sonots@gmail.com"]
|
9
|
+
s.homepage = "https://github.com/riywo/fluent-plugin-hash-forward"
|
10
|
+
s.summary = %q{Fluentd plugin to keep forwarding messsages of a specific tag pattern to a specific node}
|
11
|
+
s.description = s.summary
|
12
|
+
s.licenses = ["MIT"]
|
13
|
+
|
14
|
+
s.rubyforge_project = "fluent-plugin-hash-forward"
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
|
21
|
+
s.add_runtime_dependency "fluentd"
|
22
|
+
s.add_runtime_dependency "murmurhash3"
|
23
|
+
s.add_development_dependency "rake"
|
24
|
+
s.add_development_dependency "rspec"
|
25
|
+
s.add_development_dependency "pry"
|
26
|
+
s.add_development_dependency "pry-nav"
|
27
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'fluent/plugin/out_forward'
|
2
|
+
|
3
|
+
class Fluent::HashForwardOutput < Fluent::ForwardOutput
|
4
|
+
Fluent::Plugin.register_output('hash_forward', self)
|
5
|
+
|
6
|
+
config_param :hash_key, :string, :default => nil
|
7
|
+
|
8
|
+
def configure(conf)
|
9
|
+
super
|
10
|
+
@standby_nodes, @regular_nodes = @nodes.partition {|n| n.standby? }
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :regular_nodes
|
14
|
+
attr_reader :standby_nodes
|
15
|
+
|
16
|
+
# Override
|
17
|
+
def write_objects(tag, chunk)
|
18
|
+
return if chunk.empty?
|
19
|
+
|
20
|
+
error = nil
|
21
|
+
|
22
|
+
nodes = nodes(tag)
|
23
|
+
|
24
|
+
# below is just copy from out_forward
|
25
|
+
nodes.each do |node|
|
26
|
+
if node.available?
|
27
|
+
begin
|
28
|
+
send_data(node, tag, chunk)
|
29
|
+
return
|
30
|
+
rescue
|
31
|
+
# for load balancing during detecting crashed servers
|
32
|
+
error = $! # use the latest error
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
if error
|
38
|
+
raise error
|
39
|
+
else
|
40
|
+
raise "no nodes are available" # TODO message
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# Override: I don't use weight
|
45
|
+
def rebuild_weight_array
|
46
|
+
end
|
47
|
+
|
48
|
+
# Get nodes (a regular_node and a standby_node if available) using hash algorithm
|
49
|
+
def nodes(tag)
|
50
|
+
hash_key = @hash_key ? expand_placeholder(@hash_key, tag) : tag
|
51
|
+
regular_index = get_index(hash_key, regular_nodes.size)
|
52
|
+
standby_index = standby_nodes.size > 0 ? get_index(hash_key, standby_nodes.size) : 0
|
53
|
+
[regular_nodes[regular_index], standby_nodes[standby_index]].compact
|
54
|
+
end
|
55
|
+
|
56
|
+
# hashing(key) mod N
|
57
|
+
def get_index(key, size)
|
58
|
+
require 'murmurhash3'
|
59
|
+
MurmurHash3::V32.str_hash(key) % size
|
60
|
+
end
|
61
|
+
|
62
|
+
# Replace ${tag} and ${tags} placeholders in a string
|
63
|
+
#
|
64
|
+
# @param [String] str the string to be expanded
|
65
|
+
# @param [String] tag tag of a message
|
66
|
+
def expand_placeholder(str, tag)
|
67
|
+
struct = UndefOpenStruct.new
|
68
|
+
struct.tag = tag
|
69
|
+
struct.tags = tag.split('.')
|
70
|
+
str = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
|
71
|
+
eval "\"#{str}\"", struct.instance_eval { binding }
|
72
|
+
end
|
73
|
+
|
74
|
+
class UndefOpenStruct < OpenStruct
|
75
|
+
(Object.instance_methods).each do |m|
|
76
|
+
undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require_relative 'spec_helper'
|
3
|
+
|
4
|
+
describe Fluent::HashForwardOutput do
|
5
|
+
before { Fluent::Test.setup }
|
6
|
+
CONFIG = %[
|
7
|
+
type hash_forward
|
8
|
+
flush_interval 1s
|
9
|
+
|
10
|
+
<server>
|
11
|
+
host 192.168.1.3
|
12
|
+
port 24224
|
13
|
+
</server>
|
14
|
+
<server>
|
15
|
+
host 192.168.1.4
|
16
|
+
port 24224
|
17
|
+
</server>
|
18
|
+
|
19
|
+
<secondary>
|
20
|
+
type file
|
21
|
+
path /var/log/fluent/forward-failed
|
22
|
+
</secondary>
|
23
|
+
]
|
24
|
+
let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::HashForwardOutput, tag).configure(config) }
|
25
|
+
|
26
|
+
describe 'test configure' do
|
27
|
+
let(:tag) { 'test.tag' }
|
28
|
+
let(:config) { CONFIG }
|
29
|
+
context 'default behavior' do
|
30
|
+
it { lambda{ driver }.should_not raise_error }
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
describe 'test hashing' do
|
35
|
+
let(:tag) { 'test.tag' }
|
36
|
+
let(:config) { CONFIG }
|
37
|
+
|
38
|
+
context 'test consistency' do
|
39
|
+
before do
|
40
|
+
@node = driver.instance.nodes(tag).first
|
41
|
+
end
|
42
|
+
it 'should forward to the same node' do
|
43
|
+
expect(driver.instance.nodes(tag).first).to eq(@node)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
context 'test distribution' do
|
48
|
+
let(:tag1) { 'test.tag1' }
|
49
|
+
let(:tag2) { 'test.tag2' }
|
50
|
+
before do
|
51
|
+
MurmurHash3::V32.stub(:str_hash).with(tag1).and_return(0)
|
52
|
+
MurmurHash3::V32.stub(:str_hash).with(tag2).and_return(1)
|
53
|
+
@node1 = driver.instance.nodes(tag1).first
|
54
|
+
end
|
55
|
+
it 'should forward to the different node' do
|
56
|
+
expect(driver.instance.nodes(tag2).first).not_to eq(@node1)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
context 'test hash_key' do
|
61
|
+
let(:tag1) { 'test.tag1' }
|
62
|
+
let(:tag2) { 'test.tag2' }
|
63
|
+
let(:config) { CONFIG + %[hash_key ${tags[0..-2]}] }
|
64
|
+
before do
|
65
|
+
@node1 = driver.instance.nodes(tag1).first
|
66
|
+
end
|
67
|
+
it 'should forward to the different node' do
|
68
|
+
expect(driver.instance.nodes(tag2).first).to eq(@node1)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
describe 'test emit' do
|
74
|
+
let(:tag) { 'test.tag' }
|
75
|
+
let(:time) { Time.now.to_i }
|
76
|
+
let(:es) { Array.new(1) }
|
77
|
+
let(:chain) { Fluent::NullOutputChain.instance }
|
78
|
+
let(:config) { CONFIG }
|
79
|
+
|
80
|
+
context 'default behavior' do
|
81
|
+
before do
|
82
|
+
Fluent::Engine.stub(:now).and_return(time)
|
83
|
+
node = driver.instance.nodes(tag).first
|
84
|
+
driver.instance.should_receive(:send_data).with(node, tag, es)
|
85
|
+
end
|
86
|
+
it 'should forward' do
|
87
|
+
driver.instance.write_objects(tag, es)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
context 'test standby' do
|
92
|
+
let(:config) {
|
93
|
+
CONFIG + %[
|
94
|
+
<server>
|
95
|
+
host 192.168.1.5
|
96
|
+
port 24224
|
97
|
+
standby true
|
98
|
+
</server>
|
99
|
+
]
|
100
|
+
}
|
101
|
+
before do
|
102
|
+
Fluent::Engine.stub(:now).and_return(time)
|
103
|
+
regular_node = driver.instance.nodes(tag)[0]
|
104
|
+
standby_node = driver.instance.nodes(tag)[1]
|
105
|
+
regular_node.stub(:available?).and_return(false) # stub as regular node is not available
|
106
|
+
driver.instance.should_receive(:send_data).with(standby_node, tag, es)
|
107
|
+
end
|
108
|
+
it 'should forward to the standby node if regular node is not available' do
|
109
|
+
driver.instance.write_objects(tag, es)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
require 'rubygems'
|
3
|
+
require 'bundler'
|
4
|
+
Bundler.setup(:default, :test)
|
5
|
+
Bundler.require(:default, :test)
|
6
|
+
|
7
|
+
require 'fluent/test'
|
8
|
+
require 'rspec'
|
9
|
+
require 'pry'
|
10
|
+
|
11
|
+
$TESTING=true
|
12
|
+
$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
|
13
|
+
require 'fluent/plugin/out_hash_forward'
|
metadata
ADDED
@@ -0,0 +1,142 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-hash-forward
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ryosuke IWANAGA
|
8
|
+
- Naotoshi SEO
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-10-31 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fluentd
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - '>='
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - '>='
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: murmurhash3
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - '>='
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - '>='
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rake
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - '>='
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - '>='
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rspec
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - '>='
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: pry
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - '>='
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: pry-nav
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - '>='
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '0'
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - '>='
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0'
|
98
|
+
description: Fluentd plugin to keep forwarding messsages of a specific tag pattern
|
99
|
+
to a specific node
|
100
|
+
email:
|
101
|
+
- riywo.jp@gmail.com
|
102
|
+
- sonots@gmail.com
|
103
|
+
executables: []
|
104
|
+
extensions: []
|
105
|
+
extra_rdoc_files: []
|
106
|
+
files:
|
107
|
+
- .gitignore
|
108
|
+
- Gemfile
|
109
|
+
- README.md
|
110
|
+
- Rakefile
|
111
|
+
- fluent-plugin-hash-forward.gemspec
|
112
|
+
- lib/fluent/plugin/out_hash_forward.rb
|
113
|
+
- spec/out_hash_forward_spec.rb
|
114
|
+
- spec/spec_helper.rb
|
115
|
+
homepage: https://github.com/riywo/fluent-plugin-hash-forward
|
116
|
+
licenses:
|
117
|
+
- MIT
|
118
|
+
metadata: {}
|
119
|
+
post_install_message:
|
120
|
+
rdoc_options: []
|
121
|
+
require_paths:
|
122
|
+
- lib
|
123
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - '>='
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
129
|
+
requirements:
|
130
|
+
- - '>='
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: '0'
|
133
|
+
requirements: []
|
134
|
+
rubyforge_project: fluent-plugin-hash-forward
|
135
|
+
rubygems_version: 2.0.3
|
136
|
+
signing_key:
|
137
|
+
specification_version: 4
|
138
|
+
summary: Fluentd plugin to keep forwarding messsages of a specific tag pattern to
|
139
|
+
a specific node
|
140
|
+
test_files:
|
141
|
+
- spec/out_hash_forward_spec.rb
|
142
|
+
- spec/spec_helper.rb
|