fluent-plugin-histogram 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/fluent-plugin-histogram.gemspec +1 -1
- data/lib/fluent/plugin/out_histogram.rb +12 -4
- data/test/plugin/test_out_histogram.rb +55 -34
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
ZDc5YjIxNDUwZmM0MjA5N2VhMzMwNWY5NTU0ZTg5N2FmMmY2Zjk2Ng==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
NDRjZTllMzE2ZGVmMmEyMzM1YTJhZTMwOTQ3YjA3NjhmZWI3MTEwNw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NjVlM2M1ODY1NDU0NzQ2MDE0YzAzZWRiYTM3NWFiZjQwOTA0MzVjOTc5MmI3
|
10
|
+
ZjBjZGNhOWQ4YjhjOWI0YjM3YTFmMDFiNjY2ZjRiOTU2NWNkYTBjMjJmZWJj
|
11
|
+
MzM2YjY5MTEyMWMyZjBiNDU2NGNmMDhkNDI1ODQwYmE3ZmFlNDk=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
ZmUzMDY3MzVkYjY0ZmMxZmU0MmRhMzdhMDM1YTMwZDkwY2MyN2Q5OTdlZjZm
|
14
|
+
MmYyZTdiNmFhOWQzODNjNTg0YmQ0ZWYxYmI1YjFhOTczYjNmNDNiZjRhNGI2
|
15
|
+
YTBhYmMzM2RkMWUzNzBkNTMyOTM3YzZjMGI3ZDFhMmI1ZDYzZjI=
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-histogram"
|
5
|
-
gem.version = "0.1.
|
5
|
+
gem.version = "0.1.4"
|
6
6
|
gem.authors = ["Yusuke SHIMIZU"]
|
7
7
|
gem.email = "a.ryuklnm@gmail.com"
|
8
8
|
gem.description = "Combine inputs data and make histogram which helps to detect a hotspot."
|
@@ -9,12 +9,14 @@ module Fluent
|
|
9
9
|
config_param :tag, :string, :default => nil
|
10
10
|
config_param :tag_prefix, :string, :default => nil
|
11
11
|
config_param :tag_suffix, :string, :default => nil
|
12
|
+
config_param :out_include_hist, :bool, :default => true
|
12
13
|
config_param :input_tag_remove_prefix, :string, :default => nil
|
13
14
|
config_param :flush_interval, :time, :default => 60
|
14
15
|
config_param :count_key, :string, :default => 'keys'
|
15
16
|
config_param :bin_num, :integer, :default => 100
|
16
|
-
config_param :alpha, :integer, :default =>
|
17
|
+
config_param :alpha, :integer, :default => 0
|
17
18
|
config_param :sampling_rate, :integer, :default => 10
|
19
|
+
config_param :disable_revalue, :bool, :default => false
|
18
20
|
|
19
21
|
include Fluent::Mixin::ConfigPlaceholders
|
20
22
|
|
@@ -50,6 +52,12 @@ module Fluent
|
|
50
52
|
@sampling_counter = 0
|
51
53
|
@tick = @sampling ? @sampling_rate.to_i : 1
|
52
54
|
|
55
|
+
if @alpha > 0
|
56
|
+
@revalue = (@alpha+1)**2 if @alpha != 0
|
57
|
+
else
|
58
|
+
@disable_revalue = true
|
59
|
+
end
|
60
|
+
|
53
61
|
@mutex = Mutex.new
|
54
62
|
|
55
63
|
end
|
@@ -157,9 +165,9 @@ module Fluent
|
|
157
165
|
sigmas = map { |n| (avg - n)**2 }
|
158
166
|
Math.sqrt(sigmas.inject(:+) / size)
|
159
167
|
end
|
160
|
-
output[tag][:hist] = hist
|
161
|
-
output[tag][:sum] = sum
|
162
|
-
output[tag][:avg] = avg
|
168
|
+
output[tag][:hist] = hist if @out_include_hist
|
169
|
+
output[tag][:sum] = @disable_revalue ? sum : sum / @revalue
|
170
|
+
output[tag][:avg] = @disable_revalue ? avg : avg / @revalue
|
163
171
|
output[tag][:sd] = sd.to_i
|
164
172
|
end
|
165
173
|
output
|
@@ -31,38 +31,38 @@ class HistogramOutputTest < Test::Unit::TestCase
|
|
31
31
|
def test_small_increment_no_alpha
|
32
32
|
bin_num = 100
|
33
33
|
alpha = 0
|
34
|
-
f = create_driver(%[
|
34
|
+
f = create_driver(%[
|
35
35
|
bin_num #{bin_num}
|
36
36
|
alpha #{alpha}])
|
37
37
|
f.instance.increment("test.input", "A")
|
38
38
|
f.instance.increment("test.input", "B")
|
39
|
-
|
39
|
+
hist = f.instance.zero_hist.dup
|
40
40
|
id = "A".hash % bin_num
|
41
|
-
|
41
|
+
hist[id] += 1
|
42
42
|
id = "B".hash % bin_num
|
43
|
-
|
44
|
-
assert_equal({"test.input" => {:hist =>
|
43
|
+
hist[id] += 1
|
44
|
+
assert_equal({"test.input" => {:hist => hist, :sum => 2, :avg => 2/bin_num, :sd=>0}},
|
45
45
|
f.instance.flush)
|
46
46
|
end
|
47
47
|
|
48
48
|
def test_small_increment_with_alpha
|
49
49
|
bin_num = 100
|
50
50
|
alpha = 1
|
51
|
-
f = create_driver(%[
|
51
|
+
f = create_driver(%[
|
52
52
|
bin_num #{bin_num}
|
53
53
|
alpha #{alpha}])
|
54
54
|
f.instance.increment("test.input", "A")
|
55
55
|
f.instance.increment("test.input", "B")
|
56
|
-
|
56
|
+
hist = f.instance.zero_hist.dup
|
57
57
|
id = "A".hash % bin_num
|
58
|
-
|
59
|
-
|
60
|
-
|
58
|
+
hist[id] += 2
|
59
|
+
hist[(id + alpha) % bin_num] += 1
|
60
|
+
hist[id - alpha] += 1
|
61
61
|
id = "B".hash % bin_num
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
assert_equal({"test.input" => {:hist =>
|
62
|
+
hist[id] += 2
|
63
|
+
hist[(id + alpha) % bin_num] += 1
|
64
|
+
hist[id - alpha] += 1
|
65
|
+
assert_equal({"test.input" => {:hist => hist, :sum => 2, :avg => 2/bin_num, :sd=>0}},
|
66
66
|
f.instance.flush)
|
67
67
|
end
|
68
68
|
|
@@ -108,30 +108,30 @@ class HistogramOutputTest < Test::Unit::TestCase
|
|
108
108
|
|
109
109
|
def test_increment_sum
|
110
110
|
bin_num = 100
|
111
|
-
f = create_driver(%[
|
111
|
+
f = create_driver(%[
|
112
112
|
bin_num #{bin_num}
|
113
113
|
alpha 1 ])
|
114
114
|
1000.times do |i|
|
115
115
|
f.instance.increment("test.input", i.to_s)
|
116
116
|
end
|
117
117
|
flushed = f.instance.flush
|
118
|
-
assert_equal(1000
|
119
|
-
assert_equal(1000
|
118
|
+
assert_equal(1000, flushed["test.input"][:sum])
|
119
|
+
assert_equal(1000/bin_num, flushed["test.input"][:avg])
|
120
120
|
end
|
121
121
|
|
122
122
|
def test_emit
|
123
123
|
bin_num = 100
|
124
|
-
f = create_driver(%[
|
124
|
+
f = create_driver(%[
|
125
125
|
bin_num #{bin_num}
|
126
126
|
alpha 1 ])
|
127
127
|
f.run do
|
128
|
-
100.times do
|
128
|
+
100.times do
|
129
129
|
f.emit({"keys" => ["A", "B", "C"]})
|
130
130
|
end
|
131
131
|
end
|
132
132
|
flushed = f.instance.flush
|
133
|
-
assert_equal(300
|
134
|
-
assert_equal(300
|
133
|
+
assert_equal(300, flushed["test"][:sum])
|
134
|
+
assert_equal(300/bin_num, flushed["test"][:avg])
|
135
135
|
end
|
136
136
|
|
137
137
|
def test_some_hist_exist_case_tagging_with_emit
|
@@ -154,7 +154,7 @@ class HistogramOutputTest < Test::Unit::TestCase
|
|
154
154
|
f = create_driver(%[
|
155
155
|
count_key keys
|
156
156
|
flush_interval 10s
|
157
|
-
bin_num
|
157
|
+
bin_num #{2**10}
|
158
158
|
tag_prefix histo
|
159
159
|
tag_suffix __HOSTNAME__
|
160
160
|
hostname localhost
|
@@ -162,20 +162,20 @@ class HistogramOutputTest < Test::Unit::TestCase
|
|
162
162
|
input_tag_remove_prefix test])
|
163
163
|
# ("A".."ZZ").to_a.size == 702
|
164
164
|
data = ("A".."ZZ").to_a.shuffle
|
165
|
-
f.run do
|
166
|
-
100.times do
|
165
|
+
f.run do
|
166
|
+
100.times do
|
167
167
|
data.each_slice(10) do |d|
|
168
168
|
f.emit({"keys" => d})
|
169
169
|
end
|
170
170
|
end
|
171
171
|
end
|
172
172
|
flushed_even = f.instance.flush
|
173
|
-
|
173
|
+
|
174
174
|
#('A'..'ZZ').to_a.shuffle.size == 702
|
175
175
|
# In here, replace 7 values of ('A'..'ZZ') to 'D' as example hotspot.
|
176
176
|
data.size.times {|i| data[i] = 'D' if i%100 == 0 }
|
177
|
-
f.run do
|
178
|
-
100.times do
|
177
|
+
f.run do
|
178
|
+
100.times do
|
179
179
|
data.each_slice(10) do |d|
|
180
180
|
f.emit({"keys" => d})
|
181
181
|
end
|
@@ -183,9 +183,9 @@ class HistogramOutputTest < Test::Unit::TestCase
|
|
183
183
|
end
|
184
184
|
flushed_bias = f.instance.flush
|
185
185
|
|
186
|
-
assert_equal(true, flushed_even["histo.localhost"][:sd] < flushed_bias["histo.localhost"][:sd],
|
187
|
-
"expected
|
188
|
-
even:#{flushed_even["histo.localhost"]}
|
186
|
+
assert_equal(true, flushed_even["histo.localhost"][:sd] < flushed_bias["histo.localhost"][:sd],
|
187
|
+
"expected
|
188
|
+
even:#{flushed_even["histo.localhost"]}
|
189
189
|
<
|
190
190
|
bias:#{flushed_bias["histo.localhost"]}")
|
191
191
|
end
|
@@ -193,18 +193,18 @@ bias:#{flushed_bias["histo.localhost"]}")
|
|
193
193
|
def test_sampling
|
194
194
|
bin_num = 100
|
195
195
|
sampling_rate = 10
|
196
|
-
f = create_driver(%[
|
196
|
+
f = create_driver(%[
|
197
197
|
bin_num #{bin_num}
|
198
198
|
sampling_rate #{sampling_rate}
|
199
199
|
alpha 0 ])
|
200
200
|
f.run do
|
201
|
-
sampling_rate.times do
|
201
|
+
sampling_rate.times do
|
202
202
|
f.emit({"keys" => ["A"]})
|
203
203
|
end
|
204
204
|
end
|
205
205
|
flushed = f.instance.flush
|
206
206
|
assert_equal(sampling_rate, flushed["test"][:sum])
|
207
|
-
|
207
|
+
|
208
208
|
f.run do
|
209
209
|
1.times do # 1 < sampling_rate
|
210
210
|
f.emit({"keys" => ["A"]})
|
@@ -214,7 +214,7 @@ bias:#{flushed_bias["histo.localhost"]}")
|
|
214
214
|
assert_equal(0, flushed["test"][:sum])
|
215
215
|
|
216
216
|
f.run do
|
217
|
-
100.times do
|
217
|
+
100.times do
|
218
218
|
f.emit({"keys" => ["A", "B", "C"]})
|
219
219
|
end
|
220
220
|
end
|
@@ -222,4 +222,25 @@ bias:#{flushed_bias["histo.localhost"]}")
|
|
222
222
|
assert_equal(100*3, flushed["test"][:sum])
|
223
223
|
end
|
224
224
|
|
225
|
+
def test_revalue
|
226
|
+
f = create_driver(%[
|
227
|
+
alpha 1
|
228
|
+
disable_revalue true])
|
229
|
+
f.run do
|
230
|
+
100.times do # 1 < sampling_rate
|
231
|
+
f.emit({"keys" => ["A"]})
|
232
|
+
end
|
233
|
+
end
|
234
|
+
flushed = f.instance.flush
|
235
|
+
assert_equal(100*4, flushed["test"][:sum])
|
236
|
+
end
|
237
|
+
|
238
|
+
def test_not_include_hist
|
239
|
+
f = create_driver(%[out_include_hist false])
|
240
|
+
f.run do
|
241
|
+
f.emit({"keys" => ["A"]})
|
242
|
+
end
|
243
|
+
flushed = f.instance.flush
|
244
|
+
assert_equal(false, flushed["test"].has_key?("hist"))
|
245
|
+
end
|
225
246
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-histogram
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yusuke SHIMIZU
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-01
|
11
|
+
date: 2014-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|