fluent-plugin-histogram 0.2.1 → 0.2.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YTQwZTlkZmU3MDdjNWUxNjkwYmI5YjI5MGRiODRlNjE0ZWFjNzQ4Nw==
+    NGY2M2ZhNzY5YmE2ZDc4MzMwYTA2YTYxMmYzYWFiNDExMGQ0OWFiNA==
   data.tar.gz: !binary |-
-    NzcwMWNhM2UwNDE3MzE4NWJlOGJiYmIxNmU4YTk3MDU3MGE1ZDkyMw==
+    Y2Q3MWViYTNkMjViMzU2MDA1MGIwZTMyM2FkMzg3NjQyMThlZTdjNA==
 SHA512:
   metadata.gz: !binary |-
-    YjExZjE5ZTQzMTA1MmIxMTgzNmI1ZTViOTdmMzlhNzA2Yjg1OWNmYTQ1MmZj
-    MDk2ZDdkOTBhNDk1ZTA3NmFmZmEzMGZiMWQwZTRkOTJhYTRlMzU5NTllZjY5
-    OGM4NjE2YTExMDhiYjkwZWRlN2U2YTI4ODkxMjNlMDAzMjhlZDA=
+    MWYzNDg5ZjkyZGQyNGMzNGNjYWNiZjQ1MDc3M2NhNDNkODA3OTcxMjljYTNh
+    ZWYzOWNjOTI2YzJiYjY1OWZjOWM3OGU5YjVmOTU4NmY0M2U0MDM2MGNhMTA1
+    ODY2YTI4OWM5ZDRlN2VkOGNjN2QwOTMwOTE1OWQ0ZjhjYzkwYmQ=
   data.tar.gz: !binary |-
-    YmYzM2RjYmY0MzY5YTUxOGQ3M2U5YzFmZTBiYzg5ZDZhZTk4NjA0ZWIyMDgz
-    YzJhMWYxYmNkMzNiMWVlOTA2NDEyNmE0OWEzMTA4Njc0NmQ2YTcwOWUzNWNm
-    ZmZkMzVlMWE0YzQ4NDllZWM3ZWYyMTc2MTVjNzUxMGY3OWJlYjk=
+    Yzc0MTRkMTJkYTNkY2Y1NWY3ZjVlYWNlNDA3ZjNmMjdkNGUzYzFkMTRlYjQ2
+    YWNlNWMxZTg0ZjY5YWYyMGI1YjE2MjE1N2Q3NjZjYzcxYTNjODVlNzllN2Ez
+    ZjVhNGRiMDU5ZGM5ZWUyM2YxY2I4MmI3OTE4OWQxZTFjZTExMmU=

data/README.md

@@ -21,15 +21,19 @@ Be careful, our plugin's output histogram is not correct count-up results about
 
 if run below commands,
 ```
-$ echo '{"keys":["A",  "B",  "C",  "A"]}' | fluent-cat input.sample
-$ echo '{"keys":["A",  "B",  "D"]}' | fluent-cat input.sample
+$ echo '{"keys":"a key"}' | fluent-cat input.sample
+$ echo '{"keys":["one",  "two",  "takusan",  "takusan", "takusan", "takusan"]}' | fluent-cat input.sample
+$ echo '{"keys":{"Q":2,  "Y":2,  "X":1,  "D":1}}' | fluent-cat input.sample
 ```
 
 output is
 ```
-2014-02-01 18:18:04 +0900 histo.localhost: {
-    "hist":[0,0,0,0,3,8,8,5,3,1],
-    "sum":7,"avg":0,"sd":3}
+2014-02-02 23:08:58 +0900 histo.sample.localhost: {
+    "hist":[0,0,2,4,2,0,0,0,0,1,5,7,3,0,1,7,12,7,1,0,0,0,0,0,0,0],
+    "sum":13,
+    "avg":0,
+    "sd":3
+}
 ```
 
 count up about you specified key, and make **histogramatic something**.

data/fluent-plugin-histogram.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-histogram"
-  gem.version       = "0.2.1"
+  gem.version       = "0.2.2"
   gem.authors       = ["Yusuke SHIMIZU"]
   gem.email         = "a.ryuklnm@gmail.com"
   gem.description   = "Combine inputs data and make histogram which helps to detect a hotspot."

data/lib/fluent/plugin/out_histogram.rb

@@ -98,7 +98,7 @@ module Fluent
       hists
     end
 
-    def increment(tag, key)
+    def increment(tag, key, v=1)
       @hists[tag] ||= @zero_hist.dup
 
       # id = key.hash % @bin_num
@@ -106,7 +106,7 @@ module Fluent
       @mutex.synchronize {
         (0..@alpha).each do |alpha|
           (-alpha..alpha).each do |al|
-            @hists[tag][(id + al) % @bin_num] += @tick
+            @hists[tag][(id + al) % @bin_num] += @tick * v
           end
         end
       }
@@ -117,34 +117,48 @@ module Fluent
 
       es.each do |time, record|
         keys = record[@count_key]
-        [keys].flatten.each do |k| 
-          if !@sampling
-            increment(tag, k)
-          else
-            @sampling_counter += 1
-            if @sampling_counter >= @sampling_rate 
-              increment(tag, k) 
-              @sampling_counter = 0
+        if keys.instance_of? Hash
+          keys.each do |k, v|
+            if !@sampling
+              increment(tag, k, v)
+            else
+              @sampling_counter += v
+              if @sampling_counter >= @sampling_rate
+                increment(tag, k, v)
+                @sampling_counter = 0
+              end
+            end
+          end
+        else
+          [keys].flatten.each do |k|
+            if !@sampling
+              increment(tag, k)
+            else
+              @sampling_counter += 1
+              if @sampling_counter >= @sampling_rate
+                increment(tag, k)
+                @sampling_counter = 0
+              end
             end
           end
         end
-      end
+      end # es.each }}}
     end
-    
+
     def tagging(flushed)
       tagged = {}
       tagged = Hash[ flushed.map do |tag, hist|
         tagged_tag = tag.dup
-        if @tag 
+        if @tag
           tagged_tag = @tag
         else
           if @input_tag_remove_prefix &&
-            ( ( tag.start_with?(@remove_prefix_string) && 
+            ( ( tag.start_with?(@remove_prefix_string) &&
                tag.length > @remove_prefix_length ) ||
                tag == @input_tag_remove_prefix)
             tagged_tag = tagged_tag[@input_tag_remove_prefix.length..-1]
           end
-          
+
           tagged_tag = @tag_prefix_string + tagged_tag if @tag_prefix
           tagged_tag << @tag_suffix_string if @tag_suffix
 

data/test/plugin/test_out_histogram.rb

@@ -284,4 +284,32 @@ bias:#{flushed_bias["histo.localhost"]}")
     flushed = f.instance.flush
     assert_equal(false, flushed["test"].has_key?("hist"))
   end
+
+  def test_can_input_hash_record
+    bin_num = 10
+    f = create_driver(%[
+                        bin_num #{bin_num}
+                        alpha 0],
+                      "test.input")
+    f.run do
+      f.emit({"keys" => {"a" => 1, "b" => 2, "c" => 3}})
+    end
+    hist = f.instance.zero_hist.dup
+    id = "a"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
+    hist[id] += 1
+    id = "b"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
+    hist[id] += 2
+    id = "c"[0..9].codepoints.collect{|cp| cp}.join().to_i % bin_num
+    hist[id] += 3
+
+    sd = hist.instance_eval do
+      avg = inject(:+) / size
+      sigmas = map { |n| (avg - n)**2 }
+      Math.sqrt(sigmas.inject(:+) / size)
+    end
+    flushed = f.instance.flush
+    assert_equal({"test.input" => {:hist => hist, :sum => 6, :avg => 6/bin_num, :sd=>sd.to_i}}, flushed)
+  end
+
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-histogram
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Yusuke SHIMIZU
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-01 00:00:00.000000000 Z
+date: 2014-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler