fluent-plugin-histogram 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +18 -8
- data/bench/genload.rb +2 -2
- data/fluent-plugin-histogram.gemspec +1 -1
- data/lib/fluent/plugin/out_histogram.rb +1 -1
- data/test/plugin/test_out_histogram.rb +40 -5
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YTQwZTlkZmU3MDdjNWUxNjkwYmI5YjI5MGRiODRlNjE0ZWFjNzQ4Nw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
NzcwMWNhM2UwNDE3MzE4NWJlOGJiYmIxNmU4YTk3MDU3MGE1ZDkyMw==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
YjExZjE5ZTQzMTA1MmIxMTgzNmI1ZTViOTdmMzlhNzA2Yjg1OWNmYTQ1MmZj
|
10
|
+
MDk2ZDdkOTBhNDk1ZTA3NmFmZmEzMGZiMWQwZTRkOTJhYTRlMzU5NTllZjY5
|
11
|
+
OGM4NjE2YTExMDhiYjkwZWRlN2U2YTI4ODkxMjNlMDAzMjhlZDA=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
YmYzM2RjYmY0MzY5YTUxOGQ3M2U5YzFmZTBiYzg5ZDZhZTk4NjA0ZWIyMDgz
|
14
|
+
YzJhMWYxYmNkMzNiMWVlOTA2NDEyNmE0OWEzMTA4Njc0NmQ2YTcwOWUzNWNm
|
15
|
+
ZmZkMzVlMWE0YzQ4NDllZWM3ZWYyMTc2MTVjNzUxMGY3OWJlYjk=
|
data/README.md
CHANGED
@@ -27,7 +27,9 @@ $ echo '{"keys":["A", "B", "D"]}' | fluent-cat input.sample
|
|
27
27
|
|
28
28
|
output is
|
29
29
|
```
|
30
|
-
|
30
|
+
2014-02-01 18:18:04 +0900 histo.localhost: {
|
31
|
+
"hist":[0,0,0,0,3,8,8,5,3,1],
|
32
|
+
"sum":7,"avg":0,"sd":3}
|
31
33
|
```
|
32
34
|
|
33
35
|
count up about you specified key, and make **histogramatic something**.
|
@@ -42,17 +44,17 @@ And calculate,
|
|
42
44
|
|
43
45
|
run bench
|
44
46
|
```
|
45
|
-
$ ruby bench/genload.rb input.sample
|
47
|
+
$ ruby bench/genload.rb input.sample 7000 -l 5
|
46
48
|
```
|
47
49
|
|
48
50
|
output is,
|
49
51
|
```
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
"
|
54
|
-
"
|
55
|
-
|
52
|
+
2014-02-01 18:39:52 +0900 histo.sample.localhost: {
|
53
|
+
"hist":[0,0,0,0,0,0,0,13,36,38,31,36,37,32,32,32,30,25,10,0,0,0,0,0,0,0,0,76,221,275,248,242,274,302,293,281,274,200,66,0,0,0,0,0,0,0,0,222,655,875,899,917,907,869,851,864,859,640,230,21,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,69,203,261,189,62,0,0,3,7,6,6,7,6,7,8,7,8,10,5,0,225,681,909,873,811,820,873,870,886,923,913,902,902,921,940,900,874,892,899,901,869,839,844,827,809,805,592,192,0,0,0,51,150,193,190,198,200,201,209,203,205,168,60,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,20,22,16,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,34,104,147,118,41,0,0,1,5,9,8,4,3,7,10,8,6,5,2,0,0,0,0,0,0,0,0,0,0,0,0,5,13,16,19,21,22,23,18,20,33,31,11,0,0,0,0,0,0,0,0,35,107,152,148,124,122,146,168,172,163,117,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,9,22,29,13,0,0,0,0,0,0,1,2,1,0,1,2,1,0,0,38,102,127,100,37,0,0,1,5,9,10,11,11,11,13,14,14,10,4,2,1,1,2,1,0,0,0,0,1,2,2,7,17,24,29,34,36,29,28,41,41,25,8,1,2,1,1,2,2,2,2,30,79,97,95,99,114,122,126,132,122,93,37,2,3,3,2,2,2,3,3,2,2,2,2,1,0,0,0,1,8,22,29,20,7,2,2,2,3,5,8,10,7,3,2,1,0,1,3,3,30,94,138,112,42,4,4,11,14,9,6,5,8,12,8,3,1,1,4,8,12,13,11,10,9,9,9,8,7,3,0,7,20,27,27,22,22,29,27,25,25,16,10,8,6,4,4,7,6,4,3,18,60,94,110,121,110,90,95,116,128,98,38,12,11,11,13,7,2,5,10,11,8,5,4,3,3,8,10,6,12,21,20,19,18,14,11,10,10,13,16,13,9,9,8,5,4,5,9,11,41,95,106,70,26,7,6,4,15,26,19,12,12,14,16,17,15,10,8,6,4,5,5,3,5,8,7,8,10,9,6,7,15,22,25,24,22,29,37,35,27,16,12,11,8,10,12,16,21,18,9,23,72,107,99,108,150,158,135,139,166,148,95,69,71,59,25,10,11,9,8,7,8,13,14,12,13,10,3,1,11,28,37,35,25,15,10,7,9,15,18,13,12,23,26,18,14,15,13,10,38,83,85,54,42,58,68,52,25,15,20,19,17,19,17,12,11,11,9,13,20,22,19,12,13,24,23,15,16,14,13,18,27,38,35,26,26,32,42,47,44,37,26,17,13,16,18,13,13,16,11,29,75,95,94,108,142,171,172,155,139,121,91,73,67,52,36,28,20,11,10,17,27,26,11,4,15,31,36,31,31,37,38,34,33,32,25,19,16,13,13,13,16,22,22,20,24,34,31,14,27,71,105,98,62,57,77,66,34,26,31,28,29,38,39,26,23,30,29,27,29,27,17,10,20,35,36,25,20,24,23,20,27,41,42,33,34,41,45,44,42,40,36,32,29,28,20,20,35,41,40,35,31,34,33,31,46,61,65,74,80,81,79,84,94,77,44,31,31,33,30,17,21,33,30,28,37,51,56,43,30,32,37,34,29,32,33,32,37,39,39,41,41,37,24,20,39,54,51,37,37,51,51,67,118,159,164,148,125,105,101,100,97,102,123,144,131,110,95,81,92,98,96,112,116,107,89,83,111,133,130,126,128,125,139,162,152,131,122,111,120,148,133,100,100,107,121,133,129,115,91,89,117,136,140,139,142,174,193,182,198,206,180,174,194,192,150,126,147,169,170,151,132,129,113,293,719,955,929,902,923,907,876,884,874,676,324,161,165,149,129,99,81,92,100,98,104,121,95,66,108,144,144,139,156,188,177,151,140,115,98,119,135,132,138,137,129,137,150,141,121,126,106,76,278,646,849,716,352,157,160,161,166,182,191,172,143,155,183,179,181,188,171,157,156,153,112,100,187,231,218,229,221,208,208,216,245,271,268,247,245,254,234,221,244,245,218,203,195,188,141,154,285,327,306,417,629,755,757,732,708,707,704,643,615,556,376,280,279,267,250,256,284,266,175,220,457,581,596,617,603,551,522,550,563,553,567,565,512,476,472,468,464,454,429,423,453,447,306,95,0,0,0,0,91,283,386,287,93,0,0,4,12,14,15,20,21,16,9,7,9,9,4,0,0,0,0,0],
|
54
|
+
"sum":28415,
|
55
|
+
"avg":27,
|
56
|
+
"sd":202
|
57
|
+
}
|
56
58
|
```
|
57
59
|
|
58
60
|
## Configuration
|
@@ -65,6 +67,7 @@ output is,
|
|
65
67
|
tag_prefix histo
|
66
68
|
tag_suffix __HOSTNAME__ # this plugin mixined fluent-mixin-config-placeholders
|
67
69
|
input_tag_remove_prefix input
|
70
|
+
bin_num 1024
|
68
71
|
alpha 1 # count up like this, (■ = +1)
|
69
72
|
# ■
|
70
73
|
# ■ ■ ■ ■
|
@@ -75,6 +78,13 @@ output is,
|
|
75
78
|
</match>
|
76
79
|
```
|
77
80
|
|
81
|
+
##### Adbanced Configuration
|
82
|
+
|
83
|
+
* 'tag'
|
84
|
+
* 'out_include_histo'
|
85
|
+
* 'disable_revalue'
|
86
|
+
* 'hostname'
|
87
|
+
|
78
88
|
## Installation
|
79
89
|
|
80
90
|
Add this line to your application's Gemfile:
|
data/bench/genload.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-histogram"
|
5
|
-
gem.version = "0.2.
|
5
|
+
gem.version = "0.2.1"
|
6
6
|
gem.authors = ["Yusuke SHIMIZU"]
|
7
7
|
gem.email = "a.ryuklnm@gmail.com"
|
8
8
|
gem.description = "Combine inputs data and make histogram which helps to detect a hotspot."
|
@@ -102,7 +102,7 @@ module Fluent
|
|
102
102
|
@hists[tag] ||= @zero_hist.dup
|
103
103
|
|
104
104
|
# id = key.hash % @bin_num
|
105
|
-
id = key[0..9].codepoints.collect{|cp| cp}.join().to_i % @bin_num # attention to long key(length > 10)
|
105
|
+
id = key.to_s[0..9].codepoints.collect{|cp| cp}.join().to_i % @bin_num # attention to long key(length > 10)
|
106
106
|
@mutex.synchronize {
|
107
107
|
(0..@alpha).each do |alpha|
|
108
108
|
(-alpha..alpha).each do |al|
|
@@ -38,18 +38,32 @@ class HistogramOutputTest < Test::Unit::TestCase
|
|
38
38
|
f.run do
|
39
39
|
f.emit({"keys" => "input key"})
|
40
40
|
f.emit({"keys" => "another key"})
|
41
|
+
f.emit({"keys" => 12})
|
42
|
+
f.emit({"keys" => ["1", "2", "3"]})
|
41
43
|
end
|
42
44
|
hist = f.instance.zero_hist.dup
|
43
45
|
id = "input key"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
|
44
46
|
hist[id] += 1
|
45
47
|
id = "another key"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
|
46
48
|
hist[id] += 1
|
47
|
-
|
49
|
+
id = 12.to_s.codepoints.collect{|cp| cp}.join().to_i % bin_num
|
50
|
+
hist[id] += 1
|
51
|
+
["1", "2", "3"].each do |k|
|
52
|
+
id = k.ord % bin_num
|
53
|
+
hist[id] += 1
|
54
|
+
end
|
55
|
+
|
56
|
+
sd = hist.instance_eval do
|
57
|
+
avg = inject(:+) / size
|
58
|
+
sigmas = map { |n| (avg - n)**2 }
|
59
|
+
Math.sqrt(sigmas.inject(:+) / size)
|
60
|
+
end
|
61
|
+
assert_equal({"test.input" => {:hist => hist, :sum => 6, :avg => 6/bin_num, :sd=>sd.to_i}},
|
48
62
|
f.instance.flush)
|
49
63
|
end
|
50
64
|
|
51
65
|
def test_small_increment_with_alpha
|
52
|
-
bin_num =
|
66
|
+
bin_num = 10
|
53
67
|
alpha = 1
|
54
68
|
f = create_driver(%[
|
55
69
|
bin_num #{bin_num}
|
@@ -58,17 +72,38 @@ class HistogramOutputTest < Test::Unit::TestCase
|
|
58
72
|
f.run do
|
59
73
|
f.emit({"keys" => "A"})
|
60
74
|
f.emit({"keys" => "B"})
|
75
|
+
f.emit({"keys" => 12})
|
76
|
+
f.emit({"keys" => ["1", "2", "3"]})
|
61
77
|
end
|
62
78
|
hist = f.instance.zero_hist.dup
|
63
79
|
id = "A".ord % bin_num
|
64
80
|
hist[id] += 2
|
65
81
|
hist[(id + alpha) % bin_num] += 1
|
66
|
-
hist[id - alpha] += 1
|
82
|
+
hist[(id - alpha) % bin_num] += 1
|
83
|
+
|
67
84
|
id = "B".ord % bin_num
|
68
85
|
hist[id] += 2
|
69
86
|
hist[(id + alpha) % bin_num] += 1
|
70
|
-
hist[id - alpha] += 1
|
71
|
-
|
87
|
+
hist[(id - alpha) % bin_num] += 1
|
88
|
+
|
89
|
+
id = 12.to_s.codepoints.collect{|cp| cp}.join().to_i % bin_num
|
90
|
+
hist[id] += 2
|
91
|
+
hist[(id + alpha) % bin_num] += 1
|
92
|
+
hist[(id - alpha) % bin_num] += 1
|
93
|
+
|
94
|
+
["1", "2", "3"].each do |k|
|
95
|
+
id = k.ord % bin_num
|
96
|
+
hist[id] += 2
|
97
|
+
hist[(id + alpha) % bin_num] += 1
|
98
|
+
hist[(id - alpha) % bin_num] += 1
|
99
|
+
end
|
100
|
+
|
101
|
+
sd = hist.instance_eval do
|
102
|
+
avg = inject(:+) / size
|
103
|
+
sigmas = map { |n| (avg - n)**2 }
|
104
|
+
Math.sqrt(sigmas.inject(:+) / size)
|
105
|
+
end
|
106
|
+
assert_equal({"test.input" => {:hist => hist, :sum => 6, :avg => 6/bin_num, :sd=>sd.to_i}},
|
72
107
|
f.instance.flush)
|
73
108
|
end
|
74
109
|
|