fluent-plugin-histogram 0.2.0 → 0.2.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MTUxMjBkOGI3NTZmZGY2MzY2ZTg5MzcwODZlYjBkYjVmYmEzZjE3Mw==
+    YTQwZTlkZmU3MDdjNWUxNjkwYmI5YjI5MGRiODRlNjE0ZWFjNzQ4Nw==
   data.tar.gz: !binary |-
-    Mjc4MTFkYzMyODIwZjE5NDJjNWI2ZjFiYjk0NmViNGE3ODU1ZjNmYQ==
+    NzcwMWNhM2UwNDE3MzE4NWJlOGJiYmIxNmU4YTk3MDU3MGE1ZDkyMw==
 SHA512:
   metadata.gz: !binary |-
-    MzQwYTFiYjlhMTMzYzdlMzdkNWNmYTdhZGU2NWZkOGIwNGI2YzdkZDk2OTZj
-    NzlhNGQzYzllZDQ1MjZjMzliMzVkM2Q2ZjFhNDkxZDQxYmJiMWU2OWU3YmEy
-    MGVmNDIwYmIwNjUzNzI2MjI4Y2NiMGNlZWZiMWMxNGMyYmI3MGY=
+    YjExZjE5ZTQzMTA1MmIxMTgzNmI1ZTViOTdmMzlhNzA2Yjg1OWNmYTQ1MmZj
+    MDk2ZDdkOTBhNDk1ZTA3NmFmZmEzMGZiMWQwZTRkOTJhYTRlMzU5NTllZjY5
+    OGM4NjE2YTExMDhiYjkwZWRlN2U2YTI4ODkxMjNlMDAzMjhlZDA=
   data.tar.gz: !binary |-
-    NzQ1ZTNjMTFjNGE4ODRiNmUyZGM0MjI1ZTRmYTZhMzJhMmExNDIwY2VjZDMy
-    OGMyZTYwMmExN2UxZTBjOGIwNzgxYzU3ZWJiZjQxYWRiZTc1Yzk4MTViMjNi
-    MTNmNmI4ZThkMzg5YTdjZjQ2NWEzZjcyNmVlZmJjNjkyOTVkYTU=
+    YmYzM2RjYmY0MzY5YTUxOGQ3M2U5YzFmZTBiYzg5ZDZhZTk4NjA0ZWIyMDgz
+    YzJhMWYxYmNkMzNiMWVlOTA2NDEyNmE0OWEzMTA4Njc0NmQ2YTcwOWUzNWNm
+    ZmZkMzVlMWE0YzQ4NDllZWM3ZWYyMTc2MTVjNzUxMGY3OWJlYjk=

data/README.md

@@ -27,7 +27,9 @@ $ echo '{"keys":["A",  "B",  "D"]}' | fluent-cat input.sample
 
 output is
 ```
-2013-12-21T11:08:25+09:00       histo.sample.localhost   {"hist":[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 6, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 2, 0, 0, 0, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 0, 0, 0, 0], "sum":28, "avg":0, "sd":0}
+2014-02-01 18:18:04 +0900 histo.localhost: {
+    "hist":[0,0,0,0,3,8,8,5,3,1],
+    "sum":7,"avg":0,"sd":3}
 ```
 
 count up about you specified key, and make **histogramatic something**.
@@ -42,17 +44,17 @@ And calculate,
 
 run bench
 ```
-$ ruby bench/genload.rb input.sample 5000
+$ ruby bench/genload.rb input.sample 7000 -l 5
 ```
 
 output is,
 ```
-2013-12-21T11:09:52+09:00       histo.sample.localhost
-{"hist":
-[859, 963, 1224, 1252, 957, 764, 746, 929, 1406, 1519, 1072, 955, 1069, 916, 797, 948, 1090, 915, 727, 730, 898, 1051, 918, 780, 751, 890, 1104, 976, 949, 1138, 996, 959, 1100, 964, 840, 832, 1020, 1196, 969, 756, 750, 939, 1108, 928, 883, 1154, 1173, 951, 871, 837, 776, 896, 1048, 961, 825, 780, 959, 1113, 1034, 1019, 1090, 1274, 1370, 1207, 930, 898, 1029, 907, 951, 1113, 921, 992, 1422, 1509, 1253, 924, 941, 1099, 898, 775, 994, 1182, 1170, 1515, 1788, 1216, 870, 1038, 938, 744, 826, 969, 892, 843, 883, 840, 800, 966, 1115, 978],
-"sum":100000,
-"avg":1000,
-"sd":193}
+2014-02-01 18:39:52 +0900 histo.sample.localhost: {
+    "hist":[0,0,0,0,0,0,0,13,36,38,31,36,37,32,32,32,30,25,10,0,0,0,0,0,0,0,0,76,221,275,248,242,274,302,293,281,274,200,66,0,0,0,0,0,0,0,0,222,655,875,899,917,907,869,851,864,859,640,230,21,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,69,203,261,189,62,0,0,3,7,6,6,7,6,7,8,7,8,10,5,0,225,681,909,873,811,820,873,870,886,923,913,902,902,921,940,900,874,892,899,901,869,839,844,827,809,805,592,192,0,0,0,51,150,193,190,198,200,201,209,203,205,168,60,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,20,22,16,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,34,104,147,118,41,0,0,1,5,9,8,4,3,7,10,8,6,5,2,0,0,0,0,0,0,0,0,0,0,0,0,5,13,16,19,21,22,23,18,20,33,31,11,0,0,0,0,0,0,0,0,35,107,152,148,124,122,146,168,172,163,117,38,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,9,22,29,13,0,0,0,0,0,0,1,2,1,0,1,2,1,0,0,38,102,127,100,37,0,0,1,5,9,10,11,11,11,13,14,14,10,4,2,1,1,2,1,0,0,0,0,1,2,2,7,17,24,29,34,36,29,28,41,41,25,8,1,2,1,1,2,2,2,2,30,79,97,95,99,114,122,126,132,122,93,37,2,3,3,2,2,2,3,3,2,2,2,2,1,0,0,0,1,8,22,29,20,7,2,2,2,3,5,8,10,7,3,2,1,0,1,3,3,30,94,138,112,42,4,4,11,14,9,6,5,8,12,8,3,1,1,4,8,12,13,11,10,9,9,9,8,7,3,0,7,20,27,27,22,22,29,27,25,25,16,10,8,6,4,4,7,6,4,3,18,60,94,110,121,110,90,95,116,128,98,38,12,11,11,13,7,2,5,10,11,8,5,4,3,3,8,10,6,12,21,20,19,18,14,11,10,10,13,16,13,9,9,8,5,4,5,9,11,41,95,106,70,26,7,6,4,15,26,19,12,12,14,16,17,15,10,8,6,4,5,5,3,5,8,7,8,10,9,6,7,15,22,25,24,22,29,37,35,27,16,12,11,8,10,12,16,21,18,9,23,72,107,99,108,150,158,135,139,166,148,95,69,71,59,25,10,11,9,8,7,8,13,14,12,13,10,3,1,11,28,37,35,25,15,10,7,9,15,18,13,12,23,26,18,14,15,13,10,38,83,85,54,42,58,68,52,25,15,20,19,17,19,17,12,11,11,9,13,20,22,19,12,13,24,23,15,16,14,13,18,27,38,35,26,26,32,42,47,44,37,26,17,13,16,18,13,13,16,11,29,75,95,94,108,142,171,172,155,139,121,91,73,67,52,36,28,20,11,10,17,27,26,11,4,15,31,36,31,31,37,38,34,33,32,25,19,16,13,13,13,16,22,22,20,24,34,31,14,27,71,105,98,62,57,77,66,34,26,31,28,29,38,39,26,23,30,29,27,29,27,17,10,20,35,36,25,20,24,23,20,27,41,42,33,34,41,45,44,42,40,36,32,29,28,20,20,35,41,40,35,31,34,33,31,46,61,65,74,80,81,79,84,94,77,44,31,31,33,30,17,21,33,30,28,37,51,56,43,30,32,37,34,29,32,33,32,37,39,39,41,41,37,24,20,39,54,51,37,37,51,51,67,118,159,164,148,125,105,101,100,97,102,123,144,131,110,95,81,92,98,96,112,116,107,89,83,111,133,130,126,128,125,139,162,152,131,122,111,120,148,133,100,100,107,121,133,129,115,91,89,117,136,140,139,142,174,193,182,198,206,180,174,194,192,150,126,147,169,170,151,132,129,113,293,719,955,929,902,923,907,876,884,874,676,324,161,165,149,129,99,81,92,100,98,104,121,95,66,108,144,144,139,156,188,177,151,140,115,98,119,135,132,138,137,129,137,150,141,121,126,106,76,278,646,849,716,352,157,160,161,166,182,191,172,143,155,183,179,181,188,171,157,156,153,112,100,187,231,218,229,221,208,208,216,245,271,268,247,245,254,234,221,244,245,218,203,195,188,141,154,285,327,306,417,629,755,757,732,708,707,704,643,615,556,376,280,279,267,250,256,284,266,175,220,457,581,596,617,603,551,522,550,563,553,567,565,512,476,472,468,464,454,429,423,453,447,306,95,0,0,0,0,91,283,386,287,93,0,0,4,12,14,15,20,21,16,9,7,9,9,4,0,0,0,0,0],
+    "sum":28415,
+    "avg":27,
+    "sd":202
+}
 ```
 
 ## Configuration
@@ -65,6 +67,7 @@ output is,
     tag_prefix      histo
     tag_suffix      __HOSTNAME__    # this plugin mixined fluent-mixin-config-placeholders
     input_tag_remove_prefix input
+    bin_num         1024
     alpha           1               # count up like this,  (■ = +1)
                                     #                             ■
                                     #                  ■        ■ ■ ■
@@ -75,6 +78,13 @@ output is,
 </match>
 ```
 
+##### Adbanced Configuration
+
+* 'tag'
+* 'out_include_histo'
+* 'disable_revalue'
+* 'hostname'
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/bench/genload.rb

@@ -80,8 +80,8 @@ require 'socket'
 require 'msgpack'
 require 'benchmark'
 
-def gen_word(len=nil)
-  len = rand(5) + 1 unless len
+def gen_word(max_len=5)
+  len = rand(max_len) + 1
   rand(36**len).to_s(36)
 end
 

data/fluent-plugin-histogram.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-histogram"
-  gem.version       = "0.2.0"
+  gem.version       = "0.2.1"
   gem.authors       = ["Yusuke SHIMIZU"]
   gem.email         = "a.ryuklnm@gmail.com"
   gem.description   = "Combine inputs data and make histogram which helps to detect a hotspot."

data/lib/fluent/plugin/out_histogram.rb

@@ -102,7 +102,7 @@ module Fluent
       @hists[tag] ||= @zero_hist.dup
 
       # id = key.hash % @bin_num
-      id = key[0..9].codepoints.collect{|cp| cp}.join().to_i % @bin_num # attention to long key(length > 10)
+      id = key.to_s[0..9].codepoints.collect{|cp| cp}.join().to_i % @bin_num # attention to long key(length > 10)
       @mutex.synchronize {
         (0..@alpha).each do |alpha|
           (-alpha..alpha).each do |al|

data/test/plugin/test_out_histogram.rb

@@ -38,18 +38,32 @@ class HistogramOutputTest < Test::Unit::TestCase
     f.run do
       f.emit({"keys" => "input key"})
       f.emit({"keys" => "another key"})
+      f.emit({"keys" => 12})
+      f.emit({"keys" => ["1", "2", "3"]})
     end
     hist = f.instance.zero_hist.dup
     id = "input key"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
     hist[id] += 1
     id = "another key"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
     hist[id] += 1
-    assert_equal({"test.input" => {:hist => hist, :sum => 2, :avg => 2/bin_num, :sd=>0}},
+    id = 12.to_s.codepoints.collect{|cp| cp}.join().to_i % bin_num
+    hist[id] += 1
+    ["1", "2", "3"].each do |k|
+      id = k.ord % bin_num
+      hist[id] += 1
+    end
+
+    sd = hist.instance_eval do
+      avg = inject(:+) / size
+      sigmas = map { |n| (avg - n)**2 }
+      Math.sqrt(sigmas.inject(:+) / size)
+    end
+    assert_equal({"test.input" => {:hist => hist, :sum => 6, :avg => 6/bin_num, :sd=>sd.to_i}},
                  f.instance.flush)
   end
 
   def test_small_increment_with_alpha
-    bin_num = 100
+    bin_num = 10
     alpha = 1
     f = create_driver(%[
                         bin_num #{bin_num}
@@ -58,17 +72,38 @@ class HistogramOutputTest < Test::Unit::TestCase
     f.run do
       f.emit({"keys" => "A"})
       f.emit({"keys" => "B"})
+      f.emit({"keys" => 12})
+      f.emit({"keys" => ["1", "2", "3"]})
     end
     hist = f.instance.zero_hist.dup
     id = "A".ord % bin_num
     hist[id] += 2
     hist[(id + alpha) % bin_num] += 1
-    hist[id - alpha] += 1
+    hist[(id - alpha) % bin_num] += 1
+
     id = "B".ord % bin_num
     hist[id] += 2
     hist[(id + alpha) % bin_num] += 1
-    hist[id - alpha] += 1
-    assert_equal({"test.input" => {:hist => hist, :sum => 2, :avg => 2/bin_num, :sd=>0}},
+    hist[(id - alpha) % bin_num] += 1
+
+    id = 12.to_s.codepoints.collect{|cp| cp}.join().to_i % bin_num
+    hist[id] += 2
+    hist[(id + alpha) % bin_num] += 1
+    hist[(id - alpha) % bin_num] += 1
+
+    ["1", "2", "3"].each do |k|
+      id = k.ord % bin_num
+      hist[id] += 2
+      hist[(id + alpha) % bin_num] += 1
+      hist[(id - alpha) % bin_num] += 1
+    end
+
+    sd = hist.instance_eval do
+      avg = inject(:+) / size
+      sigmas = map { |n| (avg - n)**2 }
+      Math.sqrt(sigmas.inject(:+) / size)
+    end
+    assert_equal({"test.input" => {:hist => hist, :sum => 6, :avg => 6/bin_num, :sd=>sd.to_i}},
                  f.instance.flush)
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-histogram
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Yusuke SHIMIZU