fluent-plugin-histogram 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +9 -5
- data/fluent-plugin-histogram.gemspec +1 -1
- data/lib/fluent/plugin/out_histogram.rb +29 -15
- data/test/plugin/test_out_histogram.rb +28 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
NGY2M2ZhNzY5YmE2ZDc4MzMwYTA2YTYxMmYzYWFiNDExMGQ0OWFiNA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
Y2Q3MWViYTNkMjViMzU2MDA1MGIwZTMyM2FkMzg3NjQyMThlZTdjNA==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MWYzNDg5ZjkyZGQyNGMzNGNjYWNiZjQ1MDc3M2NhNDNkODA3OTcxMjljYTNh
|
10
|
+
ZWYzOWNjOTI2YzJiYjY1OWZjOWM3OGU5YjVmOTU4NmY0M2U0MDM2MGNhMTA1
|
11
|
+
ODY2YTI4OWM5ZDRlN2VkOGNjN2QwOTMwOTE1OWQ0ZjhjYzkwYmQ=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
Yzc0MTRkMTJkYTNkY2Y1NWY3ZjVlYWNlNDA3ZjNmMjdkNGUzYzFkMTRlYjQ2
|
14
|
+
YWNlNWMxZTg0ZjY5YWYyMGI1YjE2MjE1N2Q3NjZjYzcxYTNjODVlNzllN2Ez
|
15
|
+
ZjVhNGRiMDU5ZGM5ZWUyM2YxY2I4MmI3OTE4OWQxZTFjZTExMmU=
|
data/README.md
CHANGED
@@ -21,15 +21,19 @@ Be careful, our plugin's output histogram is not correct count-up results about
|
|
21
21
|
|
22
22
|
if run below commands,
|
23
23
|
```
|
24
|
-
$ echo '{"keys":
|
25
|
-
$ echo '{"keys":["
|
24
|
+
$ echo '{"keys":"a key"}' | fluent-cat input.sample
|
25
|
+
$ echo '{"keys":["one", "two", "takusan", "takusan", "takusan", "takusan"]}' | fluent-cat input.sample
|
26
|
+
$ echo '{"keys":{"Q":2, "Y":2, "X":1, "D":1}}' | fluent-cat input.sample
|
26
27
|
```
|
27
28
|
|
28
29
|
output is
|
29
30
|
```
|
30
|
-
2014-02-
|
31
|
-
"hist":[0,0,0,0,
|
32
|
-
"sum":
|
31
|
+
2014-02-02 23:08:58 +0900 histo.sample.localhost: {
|
32
|
+
"hist":[0,0,2,4,2,0,0,0,0,1,5,7,3,0,1,7,12,7,1,0,0,0,0,0,0,0],
|
33
|
+
"sum":13,
|
34
|
+
"avg":0,
|
35
|
+
"sd":3
|
36
|
+
}
|
33
37
|
```
|
34
38
|
|
35
39
|
count up about you specified key, and make **histogramatic something**.
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |gem|
|
4
4
|
gem.name = "fluent-plugin-histogram"
|
5
|
-
gem.version = "0.2.
|
5
|
+
gem.version = "0.2.2"
|
6
6
|
gem.authors = ["Yusuke SHIMIZU"]
|
7
7
|
gem.email = "a.ryuklnm@gmail.com"
|
8
8
|
gem.description = "Combine inputs data and make histogram which helps to detect a hotspot."
|
@@ -98,7 +98,7 @@ module Fluent
|
|
98
98
|
hists
|
99
99
|
end
|
100
100
|
|
101
|
-
def increment(tag, key)
|
101
|
+
def increment(tag, key, v=1)
|
102
102
|
@hists[tag] ||= @zero_hist.dup
|
103
103
|
|
104
104
|
# id = key.hash % @bin_num
|
@@ -106,7 +106,7 @@ module Fluent
|
|
106
106
|
@mutex.synchronize {
|
107
107
|
(0..@alpha).each do |alpha|
|
108
108
|
(-alpha..alpha).each do |al|
|
109
|
-
@hists[tag][(id + al) % @bin_num] += @tick
|
109
|
+
@hists[tag][(id + al) % @bin_num] += @tick * v
|
110
110
|
end
|
111
111
|
end
|
112
112
|
}
|
@@ -117,34 +117,48 @@ module Fluent
|
|
117
117
|
|
118
118
|
es.each do |time, record|
|
119
119
|
keys = record[@count_key]
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
120
|
+
if keys.instance_of? Hash
|
121
|
+
keys.each do |k, v|
|
122
|
+
if !@sampling
|
123
|
+
increment(tag, k, v)
|
124
|
+
else
|
125
|
+
@sampling_counter += v
|
126
|
+
if @sampling_counter >= @sampling_rate
|
127
|
+
increment(tag, k, v)
|
128
|
+
@sampling_counter = 0
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
else
|
133
|
+
[keys].flatten.each do |k|
|
134
|
+
if !@sampling
|
135
|
+
increment(tag, k)
|
136
|
+
else
|
137
|
+
@sampling_counter += 1
|
138
|
+
if @sampling_counter >= @sampling_rate
|
139
|
+
increment(tag, k)
|
140
|
+
@sampling_counter = 0
|
141
|
+
end
|
128
142
|
end
|
129
143
|
end
|
130
144
|
end
|
131
|
-
end
|
145
|
+
end # es.each }}}
|
132
146
|
end
|
133
|
-
|
147
|
+
|
134
148
|
def tagging(flushed)
|
135
149
|
tagged = {}
|
136
150
|
tagged = Hash[ flushed.map do |tag, hist|
|
137
151
|
tagged_tag = tag.dup
|
138
|
-
if @tag
|
152
|
+
if @tag
|
139
153
|
tagged_tag = @tag
|
140
154
|
else
|
141
155
|
if @input_tag_remove_prefix &&
|
142
|
-
( ( tag.start_with?(@remove_prefix_string) &&
|
156
|
+
( ( tag.start_with?(@remove_prefix_string) &&
|
143
157
|
tag.length > @remove_prefix_length ) ||
|
144
158
|
tag == @input_tag_remove_prefix)
|
145
159
|
tagged_tag = tagged_tag[@input_tag_remove_prefix.length..-1]
|
146
160
|
end
|
147
|
-
|
161
|
+
|
148
162
|
tagged_tag = @tag_prefix_string + tagged_tag if @tag_prefix
|
149
163
|
tagged_tag << @tag_suffix_string if @tag_suffix
|
150
164
|
|
@@ -284,4 +284,32 @@ bias:#{flushed_bias["histo.localhost"]}")
|
|
284
284
|
flushed = f.instance.flush
|
285
285
|
assert_equal(false, flushed["test"].has_key?("hist"))
|
286
286
|
end
|
287
|
+
|
288
|
+
def test_can_input_hash_record
|
289
|
+
bin_num = 10
|
290
|
+
f = create_driver(%[
|
291
|
+
bin_num #{bin_num}
|
292
|
+
alpha 0],
|
293
|
+
"test.input")
|
294
|
+
f.run do
|
295
|
+
f.emit({"keys" => {"a" => 1, "b" => 2, "c" => 3}})
|
296
|
+
end
|
297
|
+
hist = f.instance.zero_hist.dup
|
298
|
+
id = "a"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
|
299
|
+
hist[id] += 1
|
300
|
+
id = "b"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
|
301
|
+
hist[id] += 2
|
302
|
+
id = "c"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
|
303
|
+
hist[id] += 3
|
304
|
+
|
305
|
+
sd = hist.instance_eval do
|
306
|
+
avg = inject(:+) / size
|
307
|
+
sigmas = map { |n| (avg - n)**2 }
|
308
|
+
Math.sqrt(sigmas.inject(:+) / size)
|
309
|
+
end
|
310
|
+
flushed = f.instance.flush
|
311
|
+
assert_equal({"test.input" => {:hist => hist, :sum => 6, :avg => 6/bin_num, :sd=>sd.to_i}}, flushed)
|
312
|
+
end
|
313
|
+
|
314
|
+
|
287
315
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-histogram
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yusuke SHIMIZU
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-02-
|
11
|
+
date: 2014-02-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|