fluent-plugin-histogram 0.1.2 → 0.1.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NmJkY2ZiNTU3NjcyMzRhN2EzNzA1MWFiMzJlMjgxNGY4ODI2Mjc4Zg==
+    Yzc2ZDFiNDY3NmVlZjk5ZGE5MWU1NWFiYzRlZmM5MjcwOWYwZDIxNw==
   data.tar.gz: !binary |-
-    YjVhOGEzNWQzY2YyZTBhYjYyMjVlYjkxMDdhZDczMjg1YzA5MzQ1Nw==
+    N2E5NWRkMDBmYjIxZjc4NDcwZWI5NjA5OGM1NTMyNmY3MTAwZTg2OQ==
 SHA512:
   metadata.gz: !binary |-
-    ZDNlZGU3OWYwYzFlYjhiOTNjMzU4MjU3MTU0MzZjZmUzMzY4OGQ4NDg2ODky
-    NjBjNDJlNmM1MTE4MmE1YjcwNWNlMThlN2Y0OTQ3ZGMyN2VkNTQxMThlNmIx
-    YmUzMjQ1MzU2MDk0NzEwNjNlYmRjOGFlZjNlZTZiOTQ0YjkwMTY=
+    MDllNzkzOTE0YzE0MzUzNjE1MTQ4YTFmOGNlYzk5MWQ4NTc0YTYzMDU0NWZl
+    MWVkZDU5MzczMzMzMDE5Y2FlODVmMTM0NWJjNGE3ZmE0NWI5NmRhY2U4Njg3
+    YjYyNzc3ODFhMGNjMjI1OTk3OGIzNjMyM2VkNjk4MWFiM2E2NWE=
   data.tar.gz: !binary |-
-    YWE0MmNjOWQ2MjI5N2EzZTVmNTNiZWY1NzNkNzgzOTNkOGM2NmJmMzU0YWNh
-    OGZmNjQxNzc0OTFmZGZjZmNjYTUxMzdiNTYyOTYzNjI4NGFmYWE4MjdhMTRj
-    ZDM5NmY0MzM2ZDhmOTBlY2UzZDk3MjA2MmJiYzhkODE3ZDJiOTE=
+    ZGE4Y2MyYzM1YjBhZjc4ZjYzNzU0MmJlNTg4YTI5ZDM5Zjk1ZmJjODQ0YTVl
+    NmExZGJmNmFhOTBlMDc4YjFiOTE2MzNmOGE5NGNjOTQwMWVmZDhhYjBkYzc3
+    NTk3ZjZjNTIzOWI1YmVkZjkxMmJjMzc5YzEwZDczMjE0YzA0MmY=

data/README.md

@@ -1,25 +1,25 @@
 # fluent-plugin-histogram
 
-Fluentd output plugin. 
+Fluentd output plugin.
 
 Count up input keys, and make **scalable and rough histogram** to help detecting hotspot problems.
 
 "Scalable rough histogram" fit for cases there are an enormous variety of keys.
 
-We refered ["Strauss,  O.: Rough histograms for robust statistics, Pattern Recogniti, 2000. Proceedings. 15th International Conference on (Volume:2)"](http://ieeexplore.ieee.org/xpl/mostRecentIssue.jsp?punumber=7237) for "rough histogram". 
+We refered ["Strauss,  O.: Rough histograms for robust statistics, Pattern Recogniti, 2000. Proceedings. 15th International Conference on (Volume:2)"](http://ieeexplore.ieee.org/xpl/mostRecentIssue.jsp?punumber=7237) for "rough histogram".
 In this approarch, a increment unit is not one value(`.`), increment some values like this shape `△ `.
 To use this, please set `alpha >= 1`(default 1) option in fluent.conf.
 
 Moreover, we optimized that histogram for enormous variety of keys by fix histogram width.
-To use this, please set `bin_num`(default 100) in fluent.conf. 
+To use this, please set `bin_num`(default 100) in fluent.conf.
 
-Be careful, our plugin's output histogram is not correct count-up result about provided data. But this plugin can handle 25,000 records/sec inputs data, and that outputted histogram is enough to use for detecting hotspot problem.
+Be careful, our plugin's output histogram is not correct count-up results about provided data. But this plugin can scale out - can handle 25,000 records/sec inputs data -, and that output histogram is enough to use for detecting a hotspot problem.
 
 ## Examples
 
 ##### Example 1
 
-if run below commands, 
+if run below commands,
 ```
 $ echo '{"keys":["A",  "B",  "C",  "A"]}' | fluent-cat input.sample
 $ echo '{"keys":["A",  "B",  "D"]}' | fluent-cat input.sample
@@ -45,13 +45,13 @@ run bench
 $ ruby bench/genload.rb input.sample 5000
 ```
 
-output is, 
+output is,
 ```
-2013-12-21T11:09:52+09:00       histo.sample.localhost   
+2013-12-21T11:09:52+09:00       histo.sample.localhost
 {"hist":
-[859, 963, 1224, 1252, 957, 764, 746, 929, 1406, 1519, 1072, 955, 1069, 916, 797, 948, 1090, 915, 727, 730, 898, 1051, 918, 780, 751, 890, 1104, 976, 949, 1138, 996, 959, 1100, 964, 840, 832, 1020, 1196, 969, 756, 750, 939, 1108, 928, 883, 1154, 1173, 951, 871, 837, 776, 896, 1048, 961, 825, 780, 959, 1113, 1034, 1019, 1090, 1274, 1370, 1207, 930, 898, 1029, 907, 951, 1113, 921, 992, 1422, 1509, 1253, 924, 941, 1099, 898, 775, 994, 1182, 1170, 1515, 1788, 1216, 870, 1038, 938, 744, 826, 969, 892, 843, 883, 840, 800, 966, 1115, 978], 
-"sum":100000, 
-"avg":1000, 
+[859, 963, 1224, 1252, 957, 764, 746, 929, 1406, 1519, 1072, 955, 1069, 916, 797, 948, 1090, 915, 727, 730, 898, 1051, 918, 780, 751, 890, 1104, 976, 949, 1138, 996, 959, 1100, 964, 840, 832, 1020, 1196, 969, 756, 750, 939, 1108, 928, 883, 1154, 1173, 951, 871, 837, 776, 896, 1048, 961, 825, 780, 959, 1113, 1034, 1019, 1090, 1274, 1370, 1207, 930, 898, 1029, 907, 951, 1113, 921, 992, 1422, 1509, 1253, 924, 941, 1099, 898, 775, 994, 1182, 1170, 1515, 1788, 1216, 870, 1038, 938, 744, 826, 969, 892, 843, 883, 840, 800, 966, 1115, 978],
+"sum":100000,
+"avg":1000,
 "sd":193}
 ```
 
@@ -66,7 +66,7 @@ output is,
     tag_suffix      __HOSTNAME__    # this plugin mixined fluent-mixin-config-placeholders
     input_tag_remove_prefix input
     alpha           1               # count up like this,  (■ = +1)
-                                    #                             ■    
+                                    #                             ■
                                     #                  ■        ■ ■ ■
                                     #          ■     ■ ■ ■    ■ ■ ■ ■ ■
                                     # alpha:   0,      1,         2

data/fluent-plugin-histogram.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-histogram"
-  gem.version       = "0.1.2"
+  gem.version       = "0.1.3"
   gem.authors       = ["Yusuke SHIMIZU"]
   gem.email         = "a.ryuklnm@gmail.com"
   gem.description   = "Combine inputs data and make histogram which helps to detect a hotspot."

data/lib/fluent/plugin/out_histogram.rb

@@ -14,7 +14,7 @@ module Fluent
     config_param :count_key, :string, :default => 'keys'
     config_param :bin_num, :integer, :default => 100
     config_param :alpha, :integer, :default => 1
-    config_param :sampling_rate, :integer, :default => 1
+    config_param :sampling_rate, :integer, :default => 10
 
     include Fluent::Mixin::ConfigPlaceholders
 
@@ -32,8 +32,10 @@ module Fluent
     def configure(conf)
       super
 
-      raise Fluent::ConfigError, "bin_num must be > 0" if @bin_num <= 0
+      raise Fluent::ConfigError, 'bin_num must be > 0' if @bin_num <= 0
+      raise Fluent::ConfigError, 'sampling_rate must be >= 1' if @sampling_rate < 1
       $log.warn %Q[too small "bin_num(=#{@bin_num})" may raise unexpected outcome] if @bin_num < 100
+      @sampling = true if !!conf['sampling_rate']
 
       @tag_prefix_string = @tag_prefix + '.' if @tag_prefix
       @tag_suffix_string = '.' + @tag_suffix if @tag_suffix
@@ -46,6 +48,8 @@ module Fluent
 
       @hists = initialize_hists
       @sampling_counter = 0
+      @tick = @sampling ? @sampling_rate.to_i : 1
+
       @mutex = Mutex.new
 
     end
@@ -91,8 +95,8 @@ module Fluent
       id = key.hash % @bin_num
       @mutex.synchronize {
         (0..@alpha).each do |alpha|
-          (-alpha..alpha).each do |a|
-            @hists[tag][(id + a) % @bin_num] += 1 * @sampling_rate
+          (-alpha..alpha).each do |al|
+            @hists[tag][(id + al) % @bin_num] += @tick
           end
         end
       }
@@ -104,7 +108,7 @@ module Fluent
       es.each do |time, record|
         keys = record[@count_key]
         [keys].flatten.each do |k| 
-          if @sampling_rate == 1
+          if !@sampling
             increment(tag, k)
           else
             @sampling_counter += 1
@@ -120,27 +124,25 @@ module Fluent
     def tagging(flushed)
       tagged = {}
       tagged = Hash[ flushed.map do |tag, hist|
+        tagged_tag = tag.dup
         if @tag 
-          tag = @tag
+          tagged_tag = @tag
         else
           if @input_tag_remove_prefix &&
             ( ( tag.start_with?(@remove_prefix_string) && 
                tag.length > @remove_prefix_length ) ||
                tag == @input_tag_remove_prefix)
-            tag = tag[@input_tag_remove_prefix.length..-1]
-            tag.gsub!(/^\.|\.$/, "")
-          end
-          if @tag_prefix 
-            tag = @tag_prefix_string + tag
-            tag.gsub!(/^\.|\.$/, "")
-          end
-          if @tag_suffix
-            tag += @tag_suffix_string
-            tag.gsub!(/^\.|\.$/, "")
+            tagged_tag = tagged_tag[@input_tag_remove_prefix.length..-1]
           end
+          
+          tagged_tag = @tag_prefix_string + tagged_tag if @tag_prefix
+          tagged_tag << @tag_suffix_string if @tag_suffix
+
+          tagged_tag.gsub!(/(^\.+)|(\.+$)/, '')
+          tagged_tag.gsub!(/(\.\.+)/, '.')
         end
 
-        [tag, hist]
+        [tagged_tag, hist]
       end ]
       tagged
     end

data/test/plugin/test_out_histogram.rb

@@ -23,6 +23,9 @@ class HistogramOutputTest < Test::Unit::TestCase
     assert_raise(Fluent::ConfigError) {
       create_driver %[ bin_num 0]
     }
+    assert_raise(Fluent::ConfigError) {
+      create_driver %[ sampling_rate -1 ]
+    }
   end
 
   def test_small_increment_no_alpha
@@ -155,6 +158,7 @@ class HistogramOutputTest < Test::Unit::TestCase
                         tag_prefix     histo
                         tag_suffix     __HOSTNAME__
                         hostname       localhost
+                        alpha          1
                         input_tag_remove_prefix test])
     # ("A".."ZZ").to_a.size == 702
     data = ("A".."ZZ").to_a.shuffle
@@ -165,7 +169,7 @@ class HistogramOutputTest < Test::Unit::TestCase
         end
       end
     end
-    flushed_even = f.instance.flush["histo.localhost"]
+    flushed_even = f.instance.flush
     
     #('A'..'ZZ').to_a.shuffle.size == 702
     # In here, replace 7 values of ('A'..'ZZ') to 'D' as example hotspot.
@@ -177,26 +181,45 @@ class HistogramOutputTest < Test::Unit::TestCase
         end
       end
     end
-    flushed_uneven = f.instance.flush["histo.localhost"]
-   
-    assert_equal(true, flushed_even[:sd] < flushed_uneven[:sd])
+    flushed_bias = f.instance.flush
+
+    assert_equal(true, flushed_even["histo.localhost"][:sd] < flushed_bias["histo.localhost"][:sd], 
+                 "expected 
+even:#{flushed_even["histo.localhost"]} 
+ <
+bias:#{flushed_bias["histo.localhost"]}")
   end
 
   def test_sampling
     bin_num = 100
     sampling_rate = 10
     f = create_driver(%[ 
-                      bin_num #{bin_num}
+                      bin_num       #{bin_num}
                       sampling_rate #{sampling_rate}
                       alpha 0 ])
+    f.run do
+      sampling_rate.times do 
+        f.emit({"keys" => ["A"]})
+      end
+    end
+    flushed = f.instance.flush
+    assert_equal(sampling_rate, flushed["test"][:sum])
+ 
+    f.run do
+      1.times do  # 1 < sampling_rate
+        f.emit({"keys" => ["A"]})
+      end
+    end
+    flushed = f.instance.flush
+    assert_equal(0, flushed["test"][:sum])
+
     f.run do
       100.times do 
         f.emit({"keys" => ["A", "B", "C"]})
       end
     end
     flushed = f.instance.flush
-    assert_equal(300, flushed["test"][:sum])
-    assert_equal(300/bin_num, flushed["test"][:avg])
+    assert_equal(100*3, flushed["test"][:sum])
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-histogram
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Yusuke SHIMIZU
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-28 00:00:00.000000000 Z
+date: 2014-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler