fluent-plugin-datacounter 0.3.1 → 0.4.0

data/fluent-plugin-datacounter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-datacounter"
-  s.version     = "0.3.1"
+  s.version     = "0.4.0"
   s.authors     = ["TAGOMORI Satoshi"]
   s.email       = ["tagomoris@gmail.com"]
   s.homepage    = "https://github.com/tagomoris/fluent-plugin-datacounter"

data/lib/fluent/plugin/out_datacounter.rb

@@ -95,11 +95,12 @@ class Fluent::DataCounterOutput < Fluent::Output
 
   def count_initialized(keys=nil)
     # counts['tag'][num] = count
+    # counts['tag'][-1] = sum
     if @aggregate == :all
-      {'all' => ([0] * @patterns.length)}
+      {'all' => ([0] * (@patterns.length + 1))}
     elsif keys
       values = Array.new(keys.length) {|i|
-        Array.new(@patterns.length){|j| 0 }
+        Array.new(@patterns.length + 1){|j| 0 }
       }
       Hash[[keys, values].transpose]
     else
@@ -113,10 +114,13 @@ class Fluent::DataCounterOutput < Fluent::Output
     end
     
     @mutex.synchronize {
-      @counts[tag] ||= [0] * @patterns.length
+      @counts[tag] ||= [0] * (@patterns.length + 1)
+      sum = 0
       counts.each_with_index do |count, i|
+        sum += count
         @counts[tag][i] += count
       end
+      @counts[tag][-1] += sum
     }
   end
 
@@ -129,11 +133,11 @@ class Fluent::DataCounterOutput < Fluent::Output
 
   def generate_fields(step, target_counts, attr_prefix, output)
     sum = if @outcast_unmatched
-            target_counts[1..-1].inject(:+)
+            target_counts[1..-2].inject(:+)
           else
-            target_counts.inject(:+)
+            target_counts[-1]
           end
-    messages = sum + (@outcast_unmatched ? target_counts[0] : 0)
+    messages = target_counts.delete_at(-1)
 
     target_counts.each_with_index do |count,i|
       name = @patterns[i][1]
@@ -175,12 +179,12 @@ class Fluent::DataCounterOutput < Fluent::Output
   end
 
   def flush(step) # returns one message
-    flushed,@counts = @counts,count_initialized(@counts.keys.dup)
+    flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
     generate_output(flushed, step)
   end
 
   def flush_per_tags(step) # returns map of tag - message
-    flushed,@counts = @counts,count_initialized(@counts.keys.dup)
+    flushed,@counts = @counts,count_initialized(@counts.keys.dup.select{|k| @counts[k][-1] > 0})
     generate_output_per_tags(flushed, step)
   end
 
@@ -192,7 +196,10 @@ class Fluent::DataCounterOutput < Fluent::Output
         Fluent::Engine.emit(@tag_prefix_string + tag, time, message)
       end
     else
-      Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
+      message = flush(step)
+      if message.keys.size > 0
+        Fluent::Engine.emit(@tag, Fluent::Engine.now, message)
+      end
     end
   end
 

data/test/plugin/test_out_datacounter.rb

@@ -148,7 +148,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
       pattern1 hoge 1\d\d
       pattern2 moge 2\d\d
     ]
-    assert_equal [0,0,0], d.instance.counts['all']
+    assert_equal [0,0,0,0], d.instance.counts['all']
   end
 
   def test_countups
@@ -156,11 +156,11 @@ class DataCounterOutputTest < Test::Unit::TestCase
     assert_nil d.instance.counts['test.input']
 
     d.instance.countups('test.input', [0, 0, 0, 0, 0])
-    assert_equal [0,0,0,0,0], d.instance.counts['test.input']
+    assert_equal [0,0,0,0,0,0], d.instance.counts['test.input']
     d.instance.countups('test.input', [1, 1, 1, 0, 0])
-    assert_equal [1,1,1,0,0], d.instance.counts['test.input']
+    assert_equal [1,1,1,0,0,3], d.instance.counts['test.input']
     d.instance.countups('test.input', [0, 5, 1, 0, 0])
-    assert_equal [1,6,2,0,0], d.instance.counts['test.input']
+    assert_equal [1,6,2,0,0,9], d.instance.counts['test.input']
   end
 
   def test_stripped_tag
@@ -172,7 +172,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
 
   def test_generate_output
     d = create_driver
-    r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
+    r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
     assert_equal   60, r1['input_unmatched_count']
     assert_equal  1.0, r1['input_unmatched_rate']
     assert_equal 10.0, r1['input_unmatched_percentage']
@@ -208,7 +208,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
     assert_nil r1['input2_messages']
 
     d = create_driver (CONFIG + "\n output_messages true\n")
-    r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
+    r1 = d.instance.generate_output({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
     assert_equal  600, r1['input_messages']
     assert_equal  600, r1['input2_messages']
 
@@ -217,7 +217,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
       count_key field
       pattern1 hoge xxx\d\d
     ]
-    r2 = d.instance.generate_output({'all' => [60,240]}, 60)
+    r2 = d.instance.generate_output({'all' => [60,240,300]}, 60)
     assert_equal   60, r2['unmatched_count']
     assert_equal  1.0, r2['unmatched_rate']
     assert_equal 20.0, r2['unmatched_percentage']
@@ -231,7 +231,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
       pattern1 hoge xxx\d\d
       output_messages yes
     ]
-    r2 = d.instance.generate_output({'all' => [60,240]}, 60)
+    r2 = d.instance.generate_output({'all' => [60,240,300]}, 60)
     assert_equal   60, r2['unmatched_count']
     assert_equal  1.0, r2['unmatched_rate']
     assert_equal 20.0, r2['unmatched_percentage']
@@ -243,7 +243,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
 
   def test_generate_output_per_tag
     d = create_driver(CONFIG_OUTPUT_PER_TAG)
-    result = d.instance.generate_output_per_tags({'test.input' => [60,240,120,180,0], 'test.input2' => [0,600,0,0,0]}, 60)
+    result = d.instance.generate_output_per_tags({'test.input' => [60,240,120,180,0,600], 'test.input2' => [0,600,0,0,0,600]}, 60)
     assert_equal   60, result['input']['unmatched_count']
     assert_equal  1.0, result['input']['unmatched_rate']
     assert_equal 10.0, result['input']['unmatched_percentage']
@@ -285,7 +285,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
       output_per_tag yes
       tag_prefix d
     ]
-    r = d.instance.generate_output_per_tags({'all' => [60,240]}, 60)
+    r = d.instance.generate_output_per_tags({'all' => [60,240,300]}, 60)
     assert_equal 1, r.keys.size
     assert_equal   60, r['all']['unmatched_count']
     assert_equal  1.0, r['all']['unmatched_rate']
@@ -567,4 +567,90 @@ class DataCounterOutputTest < Test::Unit::TestCase
     assert_equal 60, data[2]['redirect_count']
     assert_equal 50.0, data[2]['redirect_percentage']
   end
+
+  def test_zero_tags
+    fields = ['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|k| 'tag1_' + k}.map{|p|
+      ['count', 'rate', 'percentage'].map{|a| p + '_' + a}
+    }.flatten
+    fields_without_percentage = ['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|k| 'tag1_' + k}.map{|p|
+      ['count', 'rate'].map{|a| p + '_' + a}
+    }.flatten
+
+    d = create_driver(CONFIG, 'test.tag1')
+    # CONFIG = %[
+    #   unit minute
+    #   aggregate tag
+    #   input_tag_remove_prefix test
+    #   count_key target
+    #   pattern1 status2xx ^2\\d\\d$
+    #   pattern2 status3xx ^3\\d\\d$
+    #   pattern3 status4xx ^4\\d\\d$
+    #   pattern4 status5xx ^5\\d\\d$
+    # ]
+    d.run do
+      60.times do
+        d.emit({'target' => '200'})
+        d.emit({'target' => '100'})
+        d.emit({'target' => '200'})
+        d.emit({'target' => '400'})
+      end
+    end
+    d.instance.flush_emit(60)
+    assert_equal 1, d.emits.size
+    r1 = d.emits[0][2]
+    assert_equal fields, r1.keys
+
+    d.instance.flush_emit(60)
+    assert_equal 2, d.emits.size # +1
+    r2 = d.emits[1][2]
+    assert_equal fields_without_percentage, r2.keys
+    assert_equal [0]*10, r2.values
+
+    d.instance.flush_emit(60)
+    assert_equal 2, d.emits.size # +0
+  end
+  def test_zer_tags_per_tag
+    fields = (['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|p|
+      ['count', 'rate', 'percentage'].map{|a| p + '_' + a}
+    }.flatten + ['messages']).sort
+    fields_without_percentage = (['unmatched','status2xx','status3xx','status4xx','status5xx'].map{|p|
+      ['count', 'rate'].map{|a| p + '_' + a}
+    }.flatten + ['messages']).sort
+
+    d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
+    # CONFIG_OUTPUT_PER_TAG = %[
+    #   unit minute
+    #   aggregate tag
+    #   output_per_tag yes
+    #   tag_prefix d
+    #   input_tag_remove_prefix test
+    #   count_key target
+    #   pattern1 status2xx ^2\\d\\d$
+    #   pattern2 status3xx ^3\\d\\d$
+    #   pattern3 status4xx ^4\\d\\d$
+    #   pattern4 status5xx ^5\\d\\d$
+    #   output_messages yes
+    # ]
+    d.run do
+      60.times do
+        d.emit({'target' => '200'})
+        d.emit({'target' => '100'})
+        d.emit({'target' => '200'})
+        d.emit({'target' => '400'})
+      end
+    end
+    d.instance.flush_emit(60)
+    assert_equal 1, d.emits.size
+    r1 = d.emits[0][2]
+    assert_equal fields, r1.keys.sort
+
+    d.instance.flush_emit(60)
+    assert_equal 2, d.emits.size # +1
+    r2 = d.emits[1][2]
+    assert_equal fields_without_percentage, r2.keys.sort
+    assert_equal [0]*11, r2.values # (_count, _rate)x5 + messages
+
+    d.instance.flush_emit(60)
+    assert_equal 2, d.emits.size # +0
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datacounter
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors: