fluent-plugin-hash-forward 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/fluent-plugin-hash-forward.gemspec +1 -1
- data/lib/fluent/plugin/out_hash_forward.rb +22 -6
- data/spec/out_hash_forward_spec.rb +167 -30
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a216c4741f2a87d809deb2a2dcd6006f4315056b
|
|
4
|
+
data.tar.gz: db7b1ca95255839c9a3134eacb34e3d580d75daf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bbff18a4bcb208ded9babba3f8f11992b62fa033285aa17b73f3ca0de441cfcea2b58eed8b6a4bbecf68ab2bb92250c543abf4bf0f22ac1fcc089f1f2846a285
|
|
7
|
+
data.tar.gz: afe9ed287898a602f6bacf7d1fdc18eb226587e11c6ae2d82a97957028bd0f358a2e93cc72df235b86a4720a70af06865f6d405a55a8c526a9a8faed2820116d
|
data/CHANGELOG.md
CHANGED
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |s|
|
|
5
5
|
s.name = "fluent-plugin-hash-forward"
|
|
6
|
-
s.version = "0.
|
|
6
|
+
s.version = "0.2.0"
|
|
7
7
|
s.authors = ["Ryosuke IWANAGA", "Naotoshi SEO"]
|
|
8
8
|
s.email = ["riywo.jp@gmail.com", "sonots@gmail.com"]
|
|
9
9
|
s.homepage = "https://github.com/riywo/fluent-plugin-hash-forward"
|
|
@@ -7,8 +7,6 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
|
|
|
7
7
|
|
|
8
8
|
def configure(conf)
|
|
9
9
|
super
|
|
10
|
-
@standby_nodes, @regular_nodes = @nodes.partition {|n| n.standby? }
|
|
11
|
-
|
|
12
10
|
if @hash_key_slice
|
|
13
11
|
lindex, rindex = @hash_key_slice.split('..', 2)
|
|
14
12
|
if lindex.nil? or rindex.nil? or lindex !~ /^-?\d+$/ or rindex !~ /^-?\d+$/
|
|
@@ -19,12 +17,18 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
|
|
|
19
17
|
end
|
|
20
18
|
end
|
|
21
19
|
|
|
20
|
+
@standby_nodes, @regular_nodes = @nodes.partition {|n| n.standby? }
|
|
21
|
+
@regular_weight_array = build_weight_array(@regular_nodes)
|
|
22
|
+
@standby_weight_array = build_weight_array(@standby_nodes)
|
|
23
|
+
|
|
22
24
|
@cache_nodes = {}
|
|
23
25
|
end
|
|
24
26
|
|
|
25
27
|
# for test
|
|
26
28
|
attr_reader :regular_nodes
|
|
27
29
|
attr_reader :standby_nodes
|
|
30
|
+
attr_reader :regular_weight_array
|
|
31
|
+
attr_reader :standby_weight_array
|
|
28
32
|
attr_accessor :hash_key_slice_lindex
|
|
29
33
|
attr_accessor :hash_key_slice_rindex
|
|
30
34
|
|
|
@@ -56,19 +60,31 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
|
|
|
56
60
|
end
|
|
57
61
|
end
|
|
58
62
|
|
|
59
|
-
# Override: I
|
|
63
|
+
# Override: I change weight algorithm
|
|
60
64
|
def rebuild_weight_array
|
|
61
65
|
end
|
|
62
66
|
|
|
67
|
+
def build_weight_array(nodes)
|
|
68
|
+
# below is just a partial copy from out_forward#rebuild_weight_array
|
|
69
|
+
weight_array = []
|
|
70
|
+
gcd = nodes.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }
|
|
71
|
+
nodes.each {|n|
|
|
72
|
+
(n.weight / gcd).times {
|
|
73
|
+
weight_array << n
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
weight_array
|
|
77
|
+
end
|
|
78
|
+
|
|
63
79
|
# Get nodes (a regular_node and a standby_node if available) using hash algorithm
|
|
64
80
|
def nodes(tag)
|
|
65
81
|
if nodes = @cache_nodes[tag]
|
|
66
82
|
return nodes
|
|
67
83
|
end
|
|
68
84
|
hash_key = @hash_key_slice ? perform_hash_key_slice(tag) : tag
|
|
69
|
-
regular_index = get_index(hash_key,
|
|
70
|
-
standby_index =
|
|
71
|
-
nodes = [
|
|
85
|
+
regular_index = @regular_weight_array.size > 0 ? get_index(hash_key, @regular_weight_array.size) : 0
|
|
86
|
+
standby_index = @standby_weight_array.size > 0 ? get_index(hash_key, @standby_weight_array.size) : 0
|
|
87
|
+
nodes = [@regular_weight_array[regular_index], @standby_weight_array[standby_index]].compact
|
|
72
88
|
@cache_nodes[tag] = nodes
|
|
73
89
|
end
|
|
74
90
|
|
|
@@ -21,7 +21,7 @@ describe Fluent::HashForwardOutput do
|
|
|
21
21
|
path /var/log/fluent/forward-failed
|
|
22
22
|
</secondary>
|
|
23
23
|
]
|
|
24
|
-
let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::HashForwardOutput, tag).configure(config) }
|
|
24
|
+
let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::HashForwardOutput, tag).configure(config).instance }
|
|
25
25
|
|
|
26
26
|
describe 'test configure' do
|
|
27
27
|
let(:tag) { 'test.tag' }
|
|
@@ -58,43 +58,103 @@ describe Fluent::HashForwardOutput do
|
|
|
58
58
|
let(:tag) { 'tag0.tag1' }
|
|
59
59
|
context 'larger than tags size' do
|
|
60
60
|
let(:config) { CONFIG + %[hash_key_slice 1..10] }
|
|
61
|
-
it { expect(driver.
|
|
61
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('tag1') }
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
context 'rindex is smaller than lindex' do
|
|
65
65
|
let(:config) { CONFIG + %[hash_key_slice 1..0] }
|
|
66
|
-
it { expect(driver.
|
|
66
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('') }
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
context 'rindex is -1' do
|
|
70
70
|
let(:config) { CONFIG + %[hash_key_slice 0..-1] }
|
|
71
|
-
it { expect(driver.
|
|
71
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq(tag) }
|
|
72
72
|
end
|
|
73
73
|
|
|
74
74
|
context 'rindex is -2' do
|
|
75
75
|
let(:config) { CONFIG + %[hash_key_slice 0..-2] }
|
|
76
|
-
it { expect(driver.
|
|
76
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('tag0') }
|
|
77
77
|
end
|
|
78
78
|
|
|
79
79
|
context 'rindex is large negative integer' do
|
|
80
80
|
let(:config) { CONFIG + %[hash_key_slice 0..-10] }
|
|
81
|
-
it { expect(driver.
|
|
81
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('') }
|
|
82
82
|
end
|
|
83
83
|
|
|
84
84
|
context 'lindex is -1' do
|
|
85
85
|
let(:config) { CONFIG + %[hash_key_slice -1..10] }
|
|
86
|
-
it { expect(driver.
|
|
86
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('tag1') }
|
|
87
87
|
end
|
|
88
88
|
|
|
89
89
|
context 'lindex is -2' do
|
|
90
90
|
let(:config) { CONFIG + %[hash_key_slice -2..10] }
|
|
91
|
-
it { expect(driver.
|
|
91
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq(tag) }
|
|
92
92
|
end
|
|
93
93
|
|
|
94
94
|
context 'lindex is large negatize integer' do
|
|
95
95
|
# this behavior looks wierd for me
|
|
96
96
|
let(:config) { CONFIG + %[hash_key_slice -3..10] }
|
|
97
|
-
it { expect(driver.
|
|
97
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('') }
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
describe 'test build_weight_array' do
|
|
102
|
+
let(:tag) { 'test.tag' }
|
|
103
|
+
context 'default behavior (60:60 equals to 1:1)' do
|
|
104
|
+
let(:config) { CONFIG }
|
|
105
|
+
it do
|
|
106
|
+
expect(driver.regular_weight_array.size).to eq(driver.regular_nodes.size)
|
|
107
|
+
expect(driver.regular_weight_array[0]).to eq(driver.regular_nodes[0])
|
|
108
|
+
expect(driver.regular_weight_array[1]).to eq(driver.regular_nodes[1])
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
context '100:0 equals to 1:0' do
|
|
113
|
+
let(:config) {
|
|
114
|
+
%[
|
|
115
|
+
type hash_forward
|
|
116
|
+
|
|
117
|
+
<server>
|
|
118
|
+
host 192.168.1.3
|
|
119
|
+
port 24224
|
|
120
|
+
weight 1
|
|
121
|
+
</server>
|
|
122
|
+
<server>
|
|
123
|
+
host 192.168.1.4
|
|
124
|
+
port 24224
|
|
125
|
+
weight 0
|
|
126
|
+
</server>
|
|
127
|
+
]
|
|
128
|
+
}
|
|
129
|
+
it do
|
|
130
|
+
expect(driver.regular_weight_array.size).to eq(1)
|
|
131
|
+
expect(driver.regular_weight_array.first).to eq(driver.regular_nodes.first)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
context '100:50 equals to 2:1' do
|
|
136
|
+
let(:config) {
|
|
137
|
+
%[
|
|
138
|
+
type hash_forward
|
|
139
|
+
|
|
140
|
+
<server>
|
|
141
|
+
host 192.168.1.3
|
|
142
|
+
port 24224
|
|
143
|
+
weight 100
|
|
144
|
+
</server>
|
|
145
|
+
<server>
|
|
146
|
+
host 192.168.1.4
|
|
147
|
+
port 24224
|
|
148
|
+
weight 50
|
|
149
|
+
</server>
|
|
150
|
+
]
|
|
151
|
+
}
|
|
152
|
+
it do
|
|
153
|
+
expect(driver.regular_weight_array.size).to eq(3)
|
|
154
|
+
expect(driver.regular_weight_array[0]).to eq(driver.regular_nodes[0])
|
|
155
|
+
expect(driver.regular_weight_array[1]).to eq(driver.regular_nodes[0])
|
|
156
|
+
expect(driver.regular_weight_array[2]).to eq(driver.regular_nodes[1])
|
|
157
|
+
end
|
|
98
158
|
end
|
|
99
159
|
end
|
|
100
160
|
|
|
@@ -104,10 +164,10 @@ describe Fluent::HashForwardOutput do
|
|
|
104
164
|
|
|
105
165
|
context 'test consistency' do
|
|
106
166
|
before do
|
|
107
|
-
@node = driver.
|
|
167
|
+
@node = driver.nodes(tag).first
|
|
108
168
|
end
|
|
109
169
|
it 'should forward to the same node' do
|
|
110
|
-
expect(driver.
|
|
170
|
+
expect(driver.nodes(tag).first).to eq(@node)
|
|
111
171
|
end
|
|
112
172
|
end
|
|
113
173
|
|
|
@@ -115,12 +175,12 @@ describe Fluent::HashForwardOutput do
|
|
|
115
175
|
let(:tag1) { 'test.tag1' }
|
|
116
176
|
let(:tag2) { 'test.tag2' }
|
|
117
177
|
before do
|
|
118
|
-
driver.
|
|
119
|
-
driver.
|
|
120
|
-
@node1 = driver.
|
|
178
|
+
driver.stub(:str_hash).with(tag1).and_return(0)
|
|
179
|
+
driver.stub(:str_hash).with(tag2).and_return(1)
|
|
180
|
+
@node1 = driver.nodes(tag1).first
|
|
121
181
|
end
|
|
122
182
|
it 'should forward to the different node' do
|
|
123
|
-
expect(driver.
|
|
183
|
+
expect(driver.nodes(tag2).first).not_to eq(@node1)
|
|
124
184
|
end
|
|
125
185
|
end
|
|
126
186
|
|
|
@@ -129,10 +189,10 @@ describe Fluent::HashForwardOutput do
|
|
|
129
189
|
let(:tag2) { 'test.tag2' }
|
|
130
190
|
let(:config) { CONFIG + %[hash_key_slice 0..-2] }
|
|
131
191
|
before do
|
|
132
|
-
@node1 = driver.
|
|
192
|
+
@node1 = driver.nodes(tag1).first
|
|
133
193
|
end
|
|
134
194
|
it 'should forward to the different node' do
|
|
135
|
-
expect(driver.
|
|
195
|
+
expect(driver.nodes(tag2).first).to eq(@node1)
|
|
136
196
|
end
|
|
137
197
|
end
|
|
138
198
|
end
|
|
@@ -147,33 +207,110 @@ describe Fluent::HashForwardOutput do
|
|
|
147
207
|
context 'default behavior' do
|
|
148
208
|
before do
|
|
149
209
|
Fluent::Engine.stub(:now).and_return(time)
|
|
150
|
-
node = driver.
|
|
151
|
-
driver.
|
|
210
|
+
node = driver.nodes(tag).first
|
|
211
|
+
driver.should_receive(:send_data).with(node, tag, es)
|
|
152
212
|
end
|
|
153
213
|
it 'should forward' do
|
|
154
|
-
driver.
|
|
214
|
+
driver.write_objects(tag, es)
|
|
155
215
|
end
|
|
156
216
|
end
|
|
157
217
|
|
|
158
218
|
context 'test standby' do
|
|
159
219
|
let(:config) {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
220
|
+
%[
|
|
221
|
+
type hash_forward
|
|
222
|
+
|
|
223
|
+
<server>
|
|
224
|
+
host 192.168.1.3
|
|
225
|
+
port 24224
|
|
226
|
+
</server>
|
|
227
|
+
<server>
|
|
228
|
+
host 192.168.1.4
|
|
229
|
+
port 24224
|
|
230
|
+
standby true
|
|
231
|
+
</server>
|
|
166
232
|
]
|
|
167
233
|
}
|
|
168
234
|
before do
|
|
169
235
|
Fluent::Engine.stub(:now).and_return(time)
|
|
170
|
-
regular_node = driver.
|
|
171
|
-
standby_node = driver.
|
|
236
|
+
regular_node = driver.regular_nodes.first
|
|
237
|
+
standby_node = driver.standby_nodes.first
|
|
172
238
|
regular_node.stub(:available?).and_return(false) # stub as regular node is not available
|
|
173
|
-
driver.
|
|
239
|
+
driver.should_receive(:send_data).with(standby_node, tag, es)
|
|
174
240
|
end
|
|
175
241
|
it 'should forward to the standby node if regular node is not available' do
|
|
176
|
-
driver.
|
|
242
|
+
driver.write_objects(tag, es)
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
context 'test weight' do
|
|
247
|
+
let(:tag0) { 'test.tag0' }
|
|
248
|
+
let(:tag1) { 'test.tag1' }
|
|
249
|
+
let(:tag2) { 'test.tag2' }
|
|
250
|
+
before do
|
|
251
|
+
Fluent::Engine.stub(:now).and_return(time)
|
|
252
|
+
driver.stub(:get_index).with(tag0, 3).and_return(0)
|
|
253
|
+
driver.stub(:get_index).with(tag1, 3).and_return(1)
|
|
254
|
+
driver.stub(:get_index).with(tag2, 3).and_return(2)
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
WEIGHT_CONFIG = %[
|
|
258
|
+
type hash_forward
|
|
259
|
+
|
|
260
|
+
<server>
|
|
261
|
+
host 192.168.1.3
|
|
262
|
+
port 24224
|
|
263
|
+
weight 100
|
|
264
|
+
</server>
|
|
265
|
+
<server>
|
|
266
|
+
host 192.168.1.4
|
|
267
|
+
port 24224
|
|
268
|
+
weight 50
|
|
269
|
+
</server>
|
|
270
|
+
]
|
|
271
|
+
|
|
272
|
+
context 'test weight on regular nodes' do
|
|
273
|
+
let(:config) { WEIGHT_CONFIG }
|
|
274
|
+
before do
|
|
275
|
+
driver.should_receive(:send_data).with(driver.regular_nodes[0], tag0, es)
|
|
276
|
+
driver.should_receive(:send_data).with(driver.regular_nodes[0], tag1, es)
|
|
277
|
+
driver.should_receive(:send_data).with(driver.regular_nodes[1], tag2, es)
|
|
278
|
+
end
|
|
279
|
+
it 'should forward to regular nodes considering weights' do
|
|
280
|
+
driver.write_objects(tag0, es)
|
|
281
|
+
driver.write_objects(tag1, es)
|
|
282
|
+
driver.write_objects(tag2, es)
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
context 'test weight on standby nodes' do
|
|
287
|
+
let(:config) { WEIGHT_CONFIG + %[
|
|
288
|
+
<server>
|
|
289
|
+
host 192.168.1.3
|
|
290
|
+
port 24224
|
|
291
|
+
weight 100
|
|
292
|
+
standby true
|
|
293
|
+
</server>
|
|
294
|
+
<server>
|
|
295
|
+
host 192.168.1.4
|
|
296
|
+
port 24224
|
|
297
|
+
weight 50
|
|
298
|
+
standby true
|
|
299
|
+
</server>
|
|
300
|
+
]
|
|
301
|
+
}
|
|
302
|
+
before do
|
|
303
|
+
driver.regular_nodes[0].stub(:available?).and_return(false)
|
|
304
|
+
driver.regular_nodes[1].stub(:available?).and_return(false)
|
|
305
|
+
driver.should_receive(:send_data).with(driver.standby_nodes[0], tag0, es)
|
|
306
|
+
driver.should_receive(:send_data).with(driver.standby_nodes[0], tag1, es)
|
|
307
|
+
driver.should_receive(:send_data).with(driver.standby_nodes[1], tag2, es)
|
|
308
|
+
end
|
|
309
|
+
it 'should forward to standby nodes considering weights' do
|
|
310
|
+
driver.write_objects(tag0, es)
|
|
311
|
+
driver.write_objects(tag1, es)
|
|
312
|
+
driver.write_objects(tag2, es)
|
|
313
|
+
end
|
|
177
314
|
end
|
|
178
315
|
end
|
|
179
316
|
end
|