fluent-plugin-numeric-counter 0.1.1 → 0.2.0

data/fluent-plugin-numeric-counter.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-numeric-counter"
-  gem.version       = "0.1.1"
+  gem.version       = "0.2.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{Counts messages, with specified key and numeric value in specified range}

data/lib/fluent/plugin/out_numeric_counter.rb

@@ -105,11 +105,12 @@ class Fluent::NumericCounterOutput < Fluent::Output
 
   def count_initialized(keys=nil)
     # counts['tag'][pattern_index_num] = count
+    # counts['tag'][-1] = sum
     if @aggregate == :all
-      {'all' => Array.new(@patterns.length){|i| 0}}
+      {'all' => Array.new(@patterns.length + 1){|i| 0}}
     elsif keys
       values = Array.new(keys.length){|i|
-        Array.new(@patterns.length){|j| 0 }
+        Array.new(@patterns.length + 1){|j| 0 }
       }
       Hash[[keys, values].transpose]
     else
@@ -123,10 +124,13 @@ class Fluent::NumericCounterOutput < Fluent::Output
     end
 
     @mutex.synchronize {
-      @counts[tag] ||= [0] * @patterns.length
+      @counts[tag] ||= [0] * (@patterns.length + 1)
+      sum = 0
       counts.each_with_index do |count, i|
+        sum += count
         @counts[tag][i] += count
       end
+      @counts[tag][-1] += sum
     }
   end
 
@@ -139,11 +143,11 @@ class Fluent::NumericCounterOutput < Fluent::Output
 
   def generate_fields(step, target_counts, attr_prefix, output)
     sum = if @outcast_unmatched
-            target_counts[1..-1].inject(:+)
+            target_counts[1..-2].inject(:+)
           else
-            target_counts.inject(:+)
+            target_counts[-1]
           end
-    messages = sum + (@outcast_unmatched ? target_counts[0] : 0)
+    messages = target_counts.delete_at(-1)
 
     target_counts.each_with_index do |count,i|
       name = @patterns[i][1]
@@ -185,12 +189,12 @@ class Fluent::NumericCounterOutput < Fluent::Output
   end
 
   def flush(step) # returns one message
-    flushed,@counts = @counts,count_initialized(@counts.keys.dup)
+    flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
     generate_output(flushed, step)
   end
 
   def flush_per_tags(step) # returns map of tag - message
-    flushed,@counts = @counts,count_initialized(@counts.keys.dup)
+    flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
     generate_output_per_tags(flushed, step)
   end
 
@@ -201,7 +205,10 @@ class Fluent::NumericCounterOutput < Fluent::Output
         Fluent::Engine.emit(@tag_prefix_string + tag, time, message)
       end
     else
-      Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
+      message = flush(step)
+      if message.keys.size > 0
+        Fluent::Engine.emit(@tag, Fluent::Engine.now, message)
+      end
     end
   end
 

data/test/plugin/test_out_numeric_counter.rb

@@ -119,11 +119,11 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_nil d.instance.counts['test.input']
 
     d.instance.countups('test.input', [0, 0, 0, 0])
-    assert_equal [0,0,0,0], d.instance.counts['test.input']
+    assert_equal [0,0,0,0,0], d.instance.counts['test.input']
     d.instance.countups('test.input', [1, 1, 1, 0])
-    assert_equal [1,1,1,0], d.instance.counts['test.input']
+    assert_equal [1,1,1,0,3], d.instance.counts['test.input']
     d.instance.countups('test.input', [0, 5, 1, 0])
-    assert_equal [1,6,2,0], d.instance.counts['test.input']
+    assert_equal [1,6,2,0,9], d.instance.counts['test.input']
   end
 
   def test_generate_output
@@ -132,7 +132,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     # pattern2 u1s 100000 1000000
     # pattern3 u3s 1000000 3000000
 
-    r1 = d.instance.generate_output({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
+    r1 = d.instance.generate_output({'test.input' => [60,240,180,120,600], 'test.input2' => [0,600,0,0,600]}, 60)
     assert_equal   60, r1['input_unmatched_count']
     assert_equal  1.0, r1['input_unmatched_rate']
     assert_equal 10.0, r1['input_unmatched_percentage']
@@ -163,7 +163,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
 
     d = create_driver(CONFIG + "\n output_messages yes \n")
 
-    r1 = d.instance.generate_output({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
+    r1 = d.instance.generate_output({'test.input' => [60,240,180,120,600], 'test.input2' => [0,600,0,0,600]}, 60)
     assert_equal 600, r1['input_messages']
     assert_equal 600, r1['input2_messages']
 
@@ -174,7 +174,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
       outcast_unmatched yes
       output_messages true
     ]
-    r2 = d.instance.generate_output({'all' => [60,240]}, 60)
+    r2 = d.instance.generate_output({'all' => [60,240,300]}, 60)
     assert_equal  60, r2['unmatched_count']
     assert_equal 1.0, r2['unmatched_rate']
     assert_nil r2['unmatched_percentage']
@@ -190,7 +190,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     # pattern2 u1s 100000 1000000
     # pattern3 u3s 1000000 3000000
 
-    r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
+    r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120,600], 'test.input2' => [0,600,0,0,600]}, 60)
     assert_equal 2, r1.keys.size
 
     r = r1['input']
@@ -225,7 +225,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
 
     d = create_driver(CONFIG_OUTPUT_PER_TAG)
 
-    r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
+    r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120,600], 'test.input2' => [0,600,0,0,600]}, 60)
     assert_equal 600, r1['input']['messages']
     assert_equal 600, r1['input2']['messages']
 
@@ -236,7 +236,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
       outcast_unmatched yes
       output_messages true
     ]
-    r2 = d.instance.generate_output_per_tags({'all' => [60,240]}, 60)
+    r2 = d.instance.generate_output_per_tags({'all' => [60,240,300]}, 60)
     assert_equal  60, r2['all']['unmatched_count']
     assert_equal 1.0, r2['all']['unmatched_rate']
     assert_nil r2['all']['unmatched_percentage']
@@ -409,4 +409,91 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_equal 20, r['unmatched_percentage']
     assert_equal 300, r['messages']
   end
+
+  def test_zero_tags
+    fields = ['unmatched','u100ms','u1s','u3s'].map{|k| 'tag1_' + k}.map{|p|
+      ['count', 'rate', 'percentage'].map{|a| p + '_' + a}
+    }.flatten
+    fields_without_percentage = ['unmatched','u100ms','u1s','u3s'].map{|k| 'tag1_' + k}.map{|p|
+      ['count', 'rate'].map{|a| p + '_' + a}
+    }.flatten
+
+    d = create_driver(CONFIG, 'test.tag1')
+    # CONFIG = %[
+    #   count_interval 60
+    #   aggregate tag
+    #   input_tag_remove_prefix test
+    #   count_key target
+    #   pattern1 u100ms 0 100000
+    #   pattern2 u1s 100000 1000000
+    #   pattern3 u3s 1000000 3000000
+    # ]
+    d.run do
+      60.times do
+        d.emit({'target' =>  '50000'})
+        d.emit({'target' => '100000'})
+        d.emit({'target' => '100001'})
+        d.emit({'target' => '0.0'})
+        d.emit({'target' => '-1'})
+      end
+    end
+    d.instance.flush_emit(60)
+    assert_equal 1, d.emits.size
+    r1 = d.emits[0][2]
+    assert_equal fields, r1.keys
+
+    d.instance.flush_emit(60)
+    assert_equal 2, d.emits.size # +1
+    r2 = d.emits[1][2]
+    assert_equal fields_without_percentage, r2.keys
+    assert_equal [0]*8, r2.values
+
+    d.instance.flush_emit(60)
+    assert_equal 2, d.emits.size # +0
+  end
+
+  def test_zero_tags_per_tag
+    fields = (['unmatched','u100ms','u1s','u3s'].map{|p|
+        ['count', 'rate', 'percentage'].map{|a| p + '_' + a}
+      }.flatten + ['messages']).sort
+    fields_without_percentage = (['unmatched','u100ms','u1s','u3s'].map{|p|
+        ['count', 'rate'].map{|a| p + '_' + a}
+      }.flatten + ['messages']).sort
+
+    d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
+    # CONFIG_OUTPUT_PER_TAG = %[
+    #   count_interval 60
+    #   aggregate tag
+    #   output_per_tag true
+    #   tag_prefix n
+    #   input_tag_remove_prefix test
+    #   count_key target
+    #   pattern1 u100ms 0 100000
+    #   pattern2 u1s 100000 1000000
+    #   pattern3 u3s 1000000 3000000
+    #   output_messages true
+    # ]
+    d.run do
+      60.times do
+        d.emit({'target' =>  '50000'})
+        d.emit({'target' => '100000'})
+        d.emit({'target' => '100001'})
+        d.emit({'target' => '0.0'})
+        d.emit({'target' => '-1'})
+      end
+    end
+    d.instance.flush_emit(60)
+    assert_equal 1, d.emits.size
+    r1 = d.emits[0][2]
+    assert_equal fields, r1.keys.sort
+
+    d.instance.flush_emit(60)
+    assert_equal 2, d.emits.size # +1
+    r2 = d.emits[1][2]
+    assert_equal fields_without_percentage, r2.keys.sort
+    assert_equal [0]*9, r2.values # (_count, _rate) x4 + messages
+
+    d.instance.flush_emit(60)
+    assert_equal 2, d.emits.size # +0
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-numeric-counter
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors: