fluent-plugin-histogram 0.1.3 → 0.1.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Yzc2ZDFiNDY3NmVlZjk5ZGE5MWU1NWFiYzRlZmM5MjcwOWYwZDIxNw==
+    ZDc5YjIxNDUwZmM0MjA5N2VhMzMwNWY5NTU0ZTg5N2FmMmY2Zjk2Ng==
   data.tar.gz: !binary |-
-    N2E5NWRkMDBmYjIxZjc4NDcwZWI5NjA5OGM1NTMyNmY3MTAwZTg2OQ==
+    NDRjZTllMzE2ZGVmMmEyMzM1YTJhZTMwOTQ3YjA3NjhmZWI3MTEwNw==
 SHA512:
   metadata.gz: !binary |-
-    MDllNzkzOTE0YzE0MzUzNjE1MTQ4YTFmOGNlYzk5MWQ4NTc0YTYzMDU0NWZl
-    MWVkZDU5MzczMzMzMDE5Y2FlODVmMTM0NWJjNGE3ZmE0NWI5NmRhY2U4Njg3
-    YjYyNzc3ODFhMGNjMjI1OTk3OGIzNjMyM2VkNjk4MWFiM2E2NWE=
+    NjVlM2M1ODY1NDU0NzQ2MDE0YzAzZWRiYTM3NWFiZjQwOTA0MzVjOTc5MmI3
+    ZjBjZGNhOWQ4YjhjOWI0YjM3YTFmMDFiNjY2ZjRiOTU2NWNkYTBjMjJmZWJj
+    MzM2YjY5MTEyMWMyZjBiNDU2NGNmMDhkNDI1ODQwYmE3ZmFlNDk=
   data.tar.gz: !binary |-
-    ZGE4Y2MyYzM1YjBhZjc4ZjYzNzU0MmJlNTg4YTI5ZDM5Zjk1ZmJjODQ0YTVl
-    NmExZGJmNmFhOTBlMDc4YjFiOTE2MzNmOGE5NGNjOTQwMWVmZDhhYjBkYzc3
-    NTk3ZjZjNTIzOWI1YmVkZjkxMmJjMzc5YzEwZDczMjE0YzA0MmY=
+    ZmUzMDY3MzVkYjY0ZmMxZmU0MmRhMzdhMDM1YTMwZDkwY2MyN2Q5OTdlZjZm
+    MmYyZTdiNmFhOWQzODNjNTg0YmQ0ZWYxYmI1YjFhOTczYjNmNDNiZjRhNGI2
+    YTBhYmMzM2RkMWUzNzBkNTMyOTM3YzZjMGI3ZDFhMmI1ZDYzZjI=

data/fluent-plugin-histogram.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-histogram"
-  gem.version       = "0.1.3"
+  gem.version       = "0.1.4"
   gem.authors       = ["Yusuke SHIMIZU"]
   gem.email         = "a.ryuklnm@gmail.com"
   gem.description   = "Combine inputs data and make histogram which helps to detect a hotspot."

data/lib/fluent/plugin/out_histogram.rb

@@ -9,12 +9,14 @@ module Fluent
     config_param :tag, :string, :default => nil
     config_param :tag_prefix, :string, :default => nil
     config_param :tag_suffix, :string, :default => nil
+    config_param :out_include_hist, :bool, :default => true
     config_param :input_tag_remove_prefix, :string, :default => nil
     config_param :flush_interval, :time, :default => 60
     config_param :count_key, :string, :default => 'keys'
     config_param :bin_num, :integer, :default => 100
-    config_param :alpha, :integer, :default => 1
+    config_param :alpha, :integer, :default => 0
     config_param :sampling_rate, :integer, :default => 10
+    config_param :disable_revalue, :bool, :default => false
 
     include Fluent::Mixin::ConfigPlaceholders
 
@@ -50,6 +52,12 @@ module Fluent
       @sampling_counter = 0
       @tick = @sampling ? @sampling_rate.to_i : 1
 
+      if @alpha > 0
+        @revalue = (@alpha+1)**2 if @alpha != 0
+      else
+        @disable_revalue = true
+      end
+
       @mutex = Mutex.new
 
     end
@@ -157,9 +165,9 @@ module Fluent
           sigmas = map { |n| (avg - n)**2 }
           Math.sqrt(sigmas.inject(:+) / size)
         end
-        output[tag][:hist] = hist
-        output[tag][:sum] = sum
-        output[tag][:avg] = avg
+        output[tag][:hist] = hist if @out_include_hist
+        output[tag][:sum] = @disable_revalue ? sum : sum / @revalue
+        output[tag][:avg] = @disable_revalue ? avg : avg / @revalue
         output[tag][:sd] = sd.to_i
       end
       output

data/test/plugin/test_out_histogram.rb

@@ -31,38 +31,38 @@ class HistogramOutputTest < Test::Unit::TestCase
   def test_small_increment_no_alpha
     bin_num = 100
     alpha = 0
-    f = create_driver(%[ 
+    f = create_driver(%[
                         bin_num #{bin_num}
                         alpha #{alpha}])
     f.instance.increment("test.input", "A")
     f.instance.increment("test.input", "B")
-    zero = f.instance.zero_hist.dup
+    hist = f.instance.zero_hist.dup
     id = "A".hash % bin_num
-    zero[id] += 1
+    hist[id] += 1
     id = "B".hash % bin_num
-    zero[id] += 1
-    assert_equal({"test.input" => {:hist => zero, :sum => 2, :avg => 2/bin_num, :sd=>0}}, 
+    hist[id] += 1
+    assert_equal({"test.input" => {:hist => hist, :sum => 2, :avg => 2/bin_num, :sd=>0}},
                  f.instance.flush)
   end
 
   def test_small_increment_with_alpha
     bin_num = 100
     alpha = 1
-    f = create_driver(%[ 
+    f = create_driver(%[
                         bin_num #{bin_num}
                         alpha #{alpha}])
     f.instance.increment("test.input", "A")
     f.instance.increment("test.input", "B")
-    zero = f.instance.zero_hist.dup
+    hist = f.instance.zero_hist.dup
     id = "A".hash % bin_num
-    zero[id] += 2
-    zero[(id + alpha) % bin_num] += 1
-    zero[id - alpha] += 1
+    hist[id] += 2
+    hist[(id + alpha) % bin_num] += 1
+    hist[id - alpha] += 1
     id = "B".hash % bin_num
-    zero[id] += 2
-    zero[(id + alpha) % bin_num] += 1
-    zero[id - alpha] += 1
-    assert_equal({"test.input" => {:hist => zero, :sum => 2*3+2, :avg => (2*3+2)/bin_num, :sd=>0}}, 
+    hist[id] += 2
+    hist[(id + alpha) % bin_num] += 1
+    hist[id - alpha] += 1
+    assert_equal({"test.input" => {:hist => hist, :sum => 2, :avg => 2/bin_num, :sd=>0}},
                  f.instance.flush)
   end
 
@@ -108,30 +108,30 @@ class HistogramOutputTest < Test::Unit::TestCase
 
   def test_increment_sum
     bin_num = 100
-    f = create_driver(%[ 
+    f = create_driver(%[
                         bin_num #{bin_num}
                         alpha   1 ])
     1000.times do |i|
       f.instance.increment("test.input", i.to_s)
     end
     flushed = f.instance.flush
-    assert_equal(1000*4, flushed["test.input"][:sum])
-    assert_equal(1000*4/bin_num, flushed["test.input"][:avg])
+    assert_equal(1000, flushed["test.input"][:sum])
+    assert_equal(1000/bin_num, flushed["test.input"][:avg])
   end
 
   def test_emit
     bin_num = 100
-    f = create_driver(%[ 
+    f = create_driver(%[
                       bin_num #{bin_num}
                       alpha 1 ])
     f.run do
-      100.times do 
+      100.times do
         f.emit({"keys" => ["A", "B", "C"]})
       end
     end
     flushed = f.instance.flush
-    assert_equal(300*4, flushed["test"][:sum])
-    assert_equal(300*4/bin_num, flushed["test"][:avg])
+    assert_equal(300, flushed["test"][:sum])
+    assert_equal(300/bin_num, flushed["test"][:avg])
   end
 
   def test_some_hist_exist_case_tagging_with_emit
@@ -154,7 +154,7 @@ class HistogramOutputTest < Test::Unit::TestCase
     f = create_driver(%[
                         count_key      keys
                         flush_interval 10s
-                        bin_num        100
+                        bin_num        #{2**10}
                         tag_prefix     histo
                         tag_suffix     __HOSTNAME__
                         hostname       localhost
@@ -162,20 +162,20 @@ class HistogramOutputTest < Test::Unit::TestCase
                         input_tag_remove_prefix test])
     # ("A".."ZZ").to_a.size == 702
     data = ("A".."ZZ").to_a.shuffle
-    f.run do 
-      100.times do 
+    f.run do
+      100.times do
         data.each_slice(10) do |d|
           f.emit({"keys" => d})
         end
       end
     end
     flushed_even = f.instance.flush
-    
+
     #('A'..'ZZ').to_a.shuffle.size == 702
     # In here, replace 7 values of ('A'..'ZZ') to 'D' as example hotspot.
     data.size.times {|i| data[i] = 'D' if i%100 == 0 }
-    f.run do 
-      100.times do 
+    f.run do
+      100.times do
         data.each_slice(10) do |d|
           f.emit({"keys" =>  d})
         end
@@ -183,9 +183,9 @@ class HistogramOutputTest < Test::Unit::TestCase
     end
     flushed_bias = f.instance.flush
 
-    assert_equal(true, flushed_even["histo.localhost"][:sd] < flushed_bias["histo.localhost"][:sd], 
-                 "expected 
-even:#{flushed_even["histo.localhost"]} 
+    assert_equal(true, flushed_even["histo.localhost"][:sd] < flushed_bias["histo.localhost"][:sd],
+                 "expected
+even:#{flushed_even["histo.localhost"]}
  <
 bias:#{flushed_bias["histo.localhost"]}")
   end
@@ -193,18 +193,18 @@ bias:#{flushed_bias["histo.localhost"]}")
   def test_sampling
     bin_num = 100
     sampling_rate = 10
-    f = create_driver(%[ 
+    f = create_driver(%[
                       bin_num       #{bin_num}
                       sampling_rate #{sampling_rate}
                       alpha 0 ])
     f.run do
-      sampling_rate.times do 
+      sampling_rate.times do
         f.emit({"keys" => ["A"]})
       end
     end
     flushed = f.instance.flush
     assert_equal(sampling_rate, flushed["test"][:sum])
- 
+
     f.run do
       1.times do  # 1 < sampling_rate
         f.emit({"keys" => ["A"]})
@@ -214,7 +214,7 @@ bias:#{flushed_bias["histo.localhost"]}")
     assert_equal(0, flushed["test"][:sum])
 
     f.run do
-      100.times do 
+      100.times do
         f.emit({"keys" => ["A", "B", "C"]})
       end
     end
@@ -222,4 +222,25 @@ bias:#{flushed_bias["histo.localhost"]}")
     assert_equal(100*3, flushed["test"][:sum])
   end
 
+  def test_revalue
+    f = create_driver(%[
+                      alpha 1
+                      disable_revalue true])
+    f.run do
+      100.times do  # 1 < sampling_rate
+        f.emit({"keys" => ["A"]})
+      end
+    end
+    flushed = f.instance.flush
+    assert_equal(100*4, flushed["test"][:sum])
+  end
+
+  def test_not_include_hist
+    f = create_driver(%[out_include_hist false])
+    f.run do
+      f.emit({"keys" => ["A"]})
+    end
+    flushed = f.instance.flush
+    assert_equal(false, flushed["test"].has_key?("hist"))
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-histogram
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Yusuke SHIMIZU
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-30 00:00:00.000000000 Z
+date: 2014-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler