fluent-plugin-hash-forward 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/fluent-plugin-hash-forward.gemspec +1 -1
- data/lib/fluent/plugin/out_hash_forward.rb +22 -6
- data/spec/out_hash_forward_spec.rb +167 -30
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a216c4741f2a87d809deb2a2dcd6006f4315056b
|
|
4
|
+
data.tar.gz: db7b1ca95255839c9a3134eacb34e3d580d75daf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bbff18a4bcb208ded9babba3f8f11992b62fa033285aa17b73f3ca0de441cfcea2b58eed8b6a4bbecf68ab2bb92250c543abf4bf0f22ac1fcc089f1f2846a285
|
|
7
|
+
data.tar.gz: afe9ed287898a602f6bacf7d1fdc18eb226587e11c6ae2d82a97957028bd0f358a2e93cc72df235b86a4720a70af06865f6d405a55a8c526a9a8faed2820116d
|
data/CHANGELOG.md
CHANGED
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |s|
|
|
5
5
|
s.name = "fluent-plugin-hash-forward"
|
|
6
|
-
s.version = "0.
|
|
6
|
+
s.version = "0.2.0"
|
|
7
7
|
s.authors = ["Ryosuke IWANAGA", "Naotoshi SEO"]
|
|
8
8
|
s.email = ["riywo.jp@gmail.com", "sonots@gmail.com"]
|
|
9
9
|
s.homepage = "https://github.com/riywo/fluent-plugin-hash-forward"
|
|
@@ -7,8 +7,6 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
|
|
|
7
7
|
|
|
8
8
|
def configure(conf)
|
|
9
9
|
super
|
|
10
|
-
@standby_nodes, @regular_nodes = @nodes.partition {|n| n.standby? }
|
|
11
|
-
|
|
12
10
|
if @hash_key_slice
|
|
13
11
|
lindex, rindex = @hash_key_slice.split('..', 2)
|
|
14
12
|
if lindex.nil? or rindex.nil? or lindex !~ /^-?\d+$/ or rindex !~ /^-?\d+$/
|
|
@@ -19,12 +17,18 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
|
|
|
19
17
|
end
|
|
20
18
|
end
|
|
21
19
|
|
|
20
|
+
@standby_nodes, @regular_nodes = @nodes.partition {|n| n.standby? }
|
|
21
|
+
@regular_weight_array = build_weight_array(@regular_nodes)
|
|
22
|
+
@standby_weight_array = build_weight_array(@standby_nodes)
|
|
23
|
+
|
|
22
24
|
@cache_nodes = {}
|
|
23
25
|
end
|
|
24
26
|
|
|
25
27
|
# for test
|
|
26
28
|
attr_reader :regular_nodes
|
|
27
29
|
attr_reader :standby_nodes
|
|
30
|
+
attr_reader :regular_weight_array
|
|
31
|
+
attr_reader :standby_weight_array
|
|
28
32
|
attr_accessor :hash_key_slice_lindex
|
|
29
33
|
attr_accessor :hash_key_slice_rindex
|
|
30
34
|
|
|
@@ -56,19 +60,31 @@ class Fluent::HashForwardOutput < Fluent::ForwardOutput
|
|
|
56
60
|
end
|
|
57
61
|
end
|
|
58
62
|
|
|
59
|
-
# Override: I
|
|
63
|
+
# Override: I change weight algorithm
|
|
60
64
|
def rebuild_weight_array
|
|
61
65
|
end
|
|
62
66
|
|
|
67
|
+
def build_weight_array(nodes)
|
|
68
|
+
# below is just a partial copy from out_forward#rebuild_weight_array
|
|
69
|
+
weight_array = []
|
|
70
|
+
gcd = nodes.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }
|
|
71
|
+
nodes.each {|n|
|
|
72
|
+
(n.weight / gcd).times {
|
|
73
|
+
weight_array << n
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
weight_array
|
|
77
|
+
end
|
|
78
|
+
|
|
63
79
|
# Get nodes (a regular_node and a standby_node if available) using hash algorithm
|
|
64
80
|
def nodes(tag)
|
|
65
81
|
if nodes = @cache_nodes[tag]
|
|
66
82
|
return nodes
|
|
67
83
|
end
|
|
68
84
|
hash_key = @hash_key_slice ? perform_hash_key_slice(tag) : tag
|
|
69
|
-
regular_index = get_index(hash_key,
|
|
70
|
-
standby_index =
|
|
71
|
-
nodes = [
|
|
85
|
+
regular_index = @regular_weight_array.size > 0 ? get_index(hash_key, @regular_weight_array.size) : 0
|
|
86
|
+
standby_index = @standby_weight_array.size > 0 ? get_index(hash_key, @standby_weight_array.size) : 0
|
|
87
|
+
nodes = [@regular_weight_array[regular_index], @standby_weight_array[standby_index]].compact
|
|
72
88
|
@cache_nodes[tag] = nodes
|
|
73
89
|
end
|
|
74
90
|
|
|
@@ -21,7 +21,7 @@ describe Fluent::HashForwardOutput do
|
|
|
21
21
|
path /var/log/fluent/forward-failed
|
|
22
22
|
</secondary>
|
|
23
23
|
]
|
|
24
|
-
let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::HashForwardOutput, tag).configure(config) }
|
|
24
|
+
let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::HashForwardOutput, tag).configure(config).instance }
|
|
25
25
|
|
|
26
26
|
describe 'test configure' do
|
|
27
27
|
let(:tag) { 'test.tag' }
|
|
@@ -58,43 +58,103 @@ describe Fluent::HashForwardOutput do
|
|
|
58
58
|
let(:tag) { 'tag0.tag1' }
|
|
59
59
|
context 'larger than tags size' do
|
|
60
60
|
let(:config) { CONFIG + %[hash_key_slice 1..10] }
|
|
61
|
-
it { expect(driver.
|
|
61
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('tag1') }
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
context 'rindex is smaller than lindex' do
|
|
65
65
|
let(:config) { CONFIG + %[hash_key_slice 1..0] }
|
|
66
|
-
it { expect(driver.
|
|
66
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('') }
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
context 'rindex is -1' do
|
|
70
70
|
let(:config) { CONFIG + %[hash_key_slice 0..-1] }
|
|
71
|
-
it { expect(driver.
|
|
71
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq(tag) }
|
|
72
72
|
end
|
|
73
73
|
|
|
74
74
|
context 'rindex is -2' do
|
|
75
75
|
let(:config) { CONFIG + %[hash_key_slice 0..-2] }
|
|
76
|
-
it { expect(driver.
|
|
76
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('tag0') }
|
|
77
77
|
end
|
|
78
78
|
|
|
79
79
|
context 'rindex is large negative integer' do
|
|
80
80
|
let(:config) { CONFIG + %[hash_key_slice 0..-10] }
|
|
81
|
-
it { expect(driver.
|
|
81
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('') }
|
|
82
82
|
end
|
|
83
83
|
|
|
84
84
|
context 'lindex is -1' do
|
|
85
85
|
let(:config) { CONFIG + %[hash_key_slice -1..10] }
|
|
86
|
-
it { expect(driver.
|
|
86
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('tag1') }
|
|
87
87
|
end
|
|
88
88
|
|
|
89
89
|
context 'lindex is -2' do
|
|
90
90
|
let(:config) { CONFIG + %[hash_key_slice -2..10] }
|
|
91
|
-
it { expect(driver.
|
|
91
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq(tag) }
|
|
92
92
|
end
|
|
93
93
|
|
|
94
94
|
context 'lindex is large negatize integer' do
|
|
95
95
|
# this behavior looks wierd for me
|
|
96
96
|
let(:config) { CONFIG + %[hash_key_slice -3..10] }
|
|
97
|
-
it { expect(driver.
|
|
97
|
+
it { expect(driver.perform_hash_key_slice(tag)).to eq('') }
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
describe 'test build_weight_array' do
|
|
102
|
+
let(:tag) { 'test.tag' }
|
|
103
|
+
context 'default behavior (60:60 equals to 1:1)' do
|
|
104
|
+
let(:config) { CONFIG }
|
|
105
|
+
it do
|
|
106
|
+
expect(driver.regular_weight_array.size).to eq(driver.regular_nodes.size)
|
|
107
|
+
expect(driver.regular_weight_array[0]).to eq(driver.regular_nodes[0])
|
|
108
|
+
expect(driver.regular_weight_array[1]).to eq(driver.regular_nodes[1])
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
context '100:0 equals to 1:0' do
|
|
113
|
+
let(:config) {
|
|
114
|
+
%[
|
|
115
|
+
type hash_forward
|
|
116
|
+
|
|
117
|
+
<server>
|
|
118
|
+
host 192.168.1.3
|
|
119
|
+
port 24224
|
|
120
|
+
weight 1
|
|
121
|
+
</server>
|
|
122
|
+
<server>
|
|
123
|
+
host 192.168.1.4
|
|
124
|
+
port 24224
|
|
125
|
+
weight 0
|
|
126
|
+
</server>
|
|
127
|
+
]
|
|
128
|
+
}
|
|
129
|
+
it do
|
|
130
|
+
expect(driver.regular_weight_array.size).to eq(1)
|
|
131
|
+
expect(driver.regular_weight_array.first).to eq(driver.regular_nodes.first)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
context '100:50 equals to 2:1' do
|
|
136
|
+
let(:config) {
|
|
137
|
+
%[
|
|
138
|
+
type hash_forward
|
|
139
|
+
|
|
140
|
+
<server>
|
|
141
|
+
host 192.168.1.3
|
|
142
|
+
port 24224
|
|
143
|
+
weight 100
|
|
144
|
+
</server>
|
|
145
|
+
<server>
|
|
146
|
+
host 192.168.1.4
|
|
147
|
+
port 24224
|
|
148
|
+
weight 50
|
|
149
|
+
</server>
|
|
150
|
+
]
|
|
151
|
+
}
|
|
152
|
+
it do
|
|
153
|
+
expect(driver.regular_weight_array.size).to eq(3)
|
|
154
|
+
expect(driver.regular_weight_array[0]).to eq(driver.regular_nodes[0])
|
|
155
|
+
expect(driver.regular_weight_array[1]).to eq(driver.regular_nodes[0])
|
|
156
|
+
expect(driver.regular_weight_array[2]).to eq(driver.regular_nodes[1])
|
|
157
|
+
end
|
|
98
158
|
end
|
|
99
159
|
end
|
|
100
160
|
|
|
@@ -104,10 +164,10 @@ describe Fluent::HashForwardOutput do
|
|
|
104
164
|
|
|
105
165
|
context 'test consistency' do
|
|
106
166
|
before do
|
|
107
|
-
@node = driver.
|
|
167
|
+
@node = driver.nodes(tag).first
|
|
108
168
|
end
|
|
109
169
|
it 'should forward to the same node' do
|
|
110
|
-
expect(driver.
|
|
170
|
+
expect(driver.nodes(tag).first).to eq(@node)
|
|
111
171
|
end
|
|
112
172
|
end
|
|
113
173
|
|
|
@@ -115,12 +175,12 @@ describe Fluent::HashForwardOutput do
|
|
|
115
175
|
let(:tag1) { 'test.tag1' }
|
|
116
176
|
let(:tag2) { 'test.tag2' }
|
|
117
177
|
before do
|
|
118
|
-
driver.
|
|
119
|
-
driver.
|
|
120
|
-
@node1 = driver.
|
|
178
|
+
driver.stub(:str_hash).with(tag1).and_return(0)
|
|
179
|
+
driver.stub(:str_hash).with(tag2).and_return(1)
|
|
180
|
+
@node1 = driver.nodes(tag1).first
|
|
121
181
|
end
|
|
122
182
|
it 'should forward to the different node' do
|
|
123
|
-
expect(driver.
|
|
183
|
+
expect(driver.nodes(tag2).first).not_to eq(@node1)
|
|
124
184
|
end
|
|
125
185
|
end
|
|
126
186
|
|
|
@@ -129,10 +189,10 @@ describe Fluent::HashForwardOutput do
|
|
|
129
189
|
let(:tag2) { 'test.tag2' }
|
|
130
190
|
let(:config) { CONFIG + %[hash_key_slice 0..-2] }
|
|
131
191
|
before do
|
|
132
|
-
@node1 = driver.
|
|
192
|
+
@node1 = driver.nodes(tag1).first
|
|
133
193
|
end
|
|
134
194
|
it 'should forward to the different node' do
|
|
135
|
-
expect(driver.
|
|
195
|
+
expect(driver.nodes(tag2).first).to eq(@node1)
|
|
136
196
|
end
|
|
137
197
|
end
|
|
138
198
|
end
|
|
@@ -147,33 +207,110 @@ describe Fluent::HashForwardOutput do
|
|
|
147
207
|
context 'default behavior' do
|
|
148
208
|
before do
|
|
149
209
|
Fluent::Engine.stub(:now).and_return(time)
|
|
150
|
-
node = driver.
|
|
151
|
-
driver.
|
|
210
|
+
node = driver.nodes(tag).first
|
|
211
|
+
driver.should_receive(:send_data).with(node, tag, es)
|
|
152
212
|
end
|
|
153
213
|
it 'should forward' do
|
|
154
|
-
driver.
|
|
214
|
+
driver.write_objects(tag, es)
|
|
155
215
|
end
|
|
156
216
|
end
|
|
157
217
|
|
|
158
218
|
context 'test standby' do
|
|
159
219
|
let(:config) {
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
220
|
+
%[
|
|
221
|
+
type hash_forward
|
|
222
|
+
|
|
223
|
+
<server>
|
|
224
|
+
host 192.168.1.3
|
|
225
|
+
port 24224
|
|
226
|
+
</server>
|
|
227
|
+
<server>
|
|
228
|
+
host 192.168.1.4
|
|
229
|
+
port 24224
|
|
230
|
+
standby true
|
|
231
|
+
</server>
|
|
166
232
|
]
|
|
167
233
|
}
|
|
168
234
|
before do
|
|
169
235
|
Fluent::Engine.stub(:now).and_return(time)
|
|
170
|
-
regular_node = driver.
|
|
171
|
-
standby_node = driver.
|
|
236
|
+
regular_node = driver.regular_nodes.first
|
|
237
|
+
standby_node = driver.standby_nodes.first
|
|
172
238
|
regular_node.stub(:available?).and_return(false) # stub as regular node is not available
|
|
173
|
-
driver.
|
|
239
|
+
driver.should_receive(:send_data).with(standby_node, tag, es)
|
|
174
240
|
end
|
|
175
241
|
it 'should forward to the standby node if regular node is not available' do
|
|
176
|
-
driver.
|
|
242
|
+
driver.write_objects(tag, es)
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
context 'test weight' do
|
|
247
|
+
let(:tag0) { 'test.tag0' }
|
|
248
|
+
let(:tag1) { 'test.tag1' }
|
|
249
|
+
let(:tag2) { 'test.tag2' }
|
|
250
|
+
before do
|
|
251
|
+
Fluent::Engine.stub(:now).and_return(time)
|
|
252
|
+
driver.stub(:get_index).with(tag0, 3).and_return(0)
|
|
253
|
+
driver.stub(:get_index).with(tag1, 3).and_return(1)
|
|
254
|
+
driver.stub(:get_index).with(tag2, 3).and_return(2)
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
WEIGHT_CONFIG = %[
|
|
258
|
+
type hash_forward
|
|
259
|
+
|
|
260
|
+
<server>
|
|
261
|
+
host 192.168.1.3
|
|
262
|
+
port 24224
|
|
263
|
+
weight 100
|
|
264
|
+
</server>
|
|
265
|
+
<server>
|
|
266
|
+
host 192.168.1.4
|
|
267
|
+
port 24224
|
|
268
|
+
weight 50
|
|
269
|
+
</server>
|
|
270
|
+
]
|
|
271
|
+
|
|
272
|
+
context 'test weight on regular nodes' do
|
|
273
|
+
let(:config) { WEIGHT_CONFIG }
|
|
274
|
+
before do
|
|
275
|
+
driver.should_receive(:send_data).with(driver.regular_nodes[0], tag0, es)
|
|
276
|
+
driver.should_receive(:send_data).with(driver.regular_nodes[0], tag1, es)
|
|
277
|
+
driver.should_receive(:send_data).with(driver.regular_nodes[1], tag2, es)
|
|
278
|
+
end
|
|
279
|
+
it 'should forward to regular nodes considering weights' do
|
|
280
|
+
driver.write_objects(tag0, es)
|
|
281
|
+
driver.write_objects(tag1, es)
|
|
282
|
+
driver.write_objects(tag2, es)
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
context 'test weight on standby nodes' do
|
|
287
|
+
let(:config) { WEIGHT_CONFIG + %[
|
|
288
|
+
<server>
|
|
289
|
+
host 192.168.1.3
|
|
290
|
+
port 24224
|
|
291
|
+
weight 100
|
|
292
|
+
standby true
|
|
293
|
+
</server>
|
|
294
|
+
<server>
|
|
295
|
+
host 192.168.1.4
|
|
296
|
+
port 24224
|
|
297
|
+
weight 50
|
|
298
|
+
standby true
|
|
299
|
+
</server>
|
|
300
|
+
]
|
|
301
|
+
}
|
|
302
|
+
before do
|
|
303
|
+
driver.regular_nodes[0].stub(:available?).and_return(false)
|
|
304
|
+
driver.regular_nodes[1].stub(:available?).and_return(false)
|
|
305
|
+
driver.should_receive(:send_data).with(driver.standby_nodes[0], tag0, es)
|
|
306
|
+
driver.should_receive(:send_data).with(driver.standby_nodes[0], tag1, es)
|
|
307
|
+
driver.should_receive(:send_data).with(driver.standby_nodes[1], tag2, es)
|
|
308
|
+
end
|
|
309
|
+
it 'should forward to standby nodes considering weights' do
|
|
310
|
+
driver.write_objects(tag0, es)
|
|
311
|
+
driver.write_objects(tag1, es)
|
|
312
|
+
driver.write_objects(tag2, es)
|
|
313
|
+
end
|
|
177
314
|
end
|
|
178
315
|
end
|
|
179
316
|
end
|