fluent-plugin-datacounter 0.2.2 → 0.3.0

data/README.rdoc

@@ -62,6 +62,20 @@ HTTP status code patterns.
       pattern5 5xx ^5\d\d$
     </match>
 
+If you want not to include 'unmatched' counts into percentage, use 'outcast_unmatched' configuration:
+
+    <match accesslog.**>
+      type datacounter
+      count_key status
+      # patternX: X(1-20)
+      pattern1 2xx ^2\d\d$
+      pattern2 3xx ^3\d\d$
+      pattern3 4xx ^4\d\d$
+      pattern4 5xx ^5\d\d$
+      outcast_unmatched yes # '*_percentage' fields culculated without 'unmatched' counts, and
+                            # 'unmatched_percentage' field will not be produced
+    </match>
+
 == TODO
 
 - consider what to do next

data/fluent-plugin-datacounter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-datacounter"
-  s.version     = "0.2.2"
+  s.version     = "0.3.0"
   s.authors     = ["TAGOMORI Satoshi"]
   s.email       = ["tagomoris@gmail.com"]
   s.homepage    = "https://github.com/tagomoris/fluent-plugin-datacounter"

data/lib/fluent/plugin/out_datacounter.rb

@@ -9,6 +9,7 @@ class Fluent::DataCounterOutput < Fluent::Output
   config_param :tag, :string, :default => 'datacount'
   config_param :input_tag_remove_prefix, :string, :default => nil
   config_param :count_key, :string
+  config_param :outcast_unmatched, :bool, :default => false
 
   # pattern0 reserved as unmatched counts
   config_param :pattern1, :string # string: NAME REGEXP
@@ -120,24 +121,37 @@ class Fluent::DataCounterOutput < Fluent::Output
   def generate_output(counts, step)
     output = {}
     if @aggregate == :all
-      sum = counts['all'].inject(:+)
+      # index 0 is unmatched
+      sum = if @outcast_unmatched
+              counts['all'][1..-1].inject(:+)
+            else
+              counts['all'].inject(:+)
+            end
       counts['all'].each_with_index do |count,i|
         name = @patterns[i][1]
         output[name + '_count'] = count
         output[name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
-        output[name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
+        unless i == 0 and @outcast_unmatched
+          output[name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
+        end
       end
       return output
     end
 
     counts.keys.each do |tag|
       t = stripped_tag(tag)
-      sum = counts[tag].inject(:+)
+      sum = if @outcast_unmatched
+              counts[tag][1..-1].inject(:+)
+            else
+              counts[tag].inject(:+)
+            end
       counts[tag].each_with_index do |count,i|
         name = @patterns[i][1]
         output[t + '_' + name + '_count'] = count
         output[t + '_' + name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
-        output[t + '_' + name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
+        unless i == 0 and @outcast_unmatched
+          output[t + '_' + name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
+        end
       end
     end
     output
@@ -177,7 +191,7 @@ class Fluent::DataCounterOutput < Fluent::Output
       value = record[@count_key]
       next if value.nil?
 
-      value = value.to_s
+      value = value.to_s.force_encoding('ASCII-8BIT')
       matched = false
       @patterns.each do |index, name, regexp|
         next unless regexp and regexp.match(value)

data/test/plugin/test_out_datacounter.rb

@@ -64,6 +64,7 @@ class DataCounterOutputTest < Test::Unit::TestCase
     assert_nil d.instance.input_tag_remove_prefix
     assert_equal 'field', d.instance.count_key
     assert_equal 'ok ^2\d\d$', d.instance.pattern1
+    assert_equal false, d.instance.outcast_unmatched
 
     d1 = create_driver %[
       unit minute
@@ -95,8 +96,10 @@ class DataCounterOutputTest < Test::Unit::TestCase
       count_interval 30s
       count_key field
       pattern1 ok ^2\\d\\d$
+      outcast_unmatched yes
     ]
     assert_equal 30, d.instance.tick
+    assert_equal true, d.instance.outcast_unmatched
   end
 
   def test_count_initialized
@@ -272,6 +275,57 @@ class DataCounterOutputTest < Test::Unit::TestCase
     assert_equal 0, data[2]['unmatched_count']
     assert_equal 0.0, data[2]['unmatched_rate']
     assert_equal 0.0, data[2]['unmatched_percentage']
-  end
 
+    d3 = create_driver(%[
+      count_key target
+      input_tag_remove_prefix test
+      pattern1 ok 2\\d\\d
+      pattern2 redirect 3\\d\\d
+      outcast_unmatched yes
+    ], 'test.tag2')
+    d3.run do
+      60.times do
+        d3.emit({'target' => '200'})
+        d3.emit({'target' => '300'})
+        d3.emit({'target' => '400'})
+      end
+    end
+    d3.instance.flush_emit(180)
+    emits = d3.emits
+    assert_equal 1, emits.length
+    data = emits[0]
+    assert_equal 'datacount', data[0] # tag
+    assert_equal 60, data[2]['tag2_unmatched_count']
+    assert_nil data[2]['tag2_unmatched_percentage']
+    assert_equal 60, data[2]['tag2_ok_count']
+    assert_equal 50.0, data[2]['tag2_ok_percentage']
+    assert_equal 60, data[2]['tag2_redirect_count']
+    assert_equal 50.0, data[2]['tag2_redirect_percentage']
+
+    d3 = create_driver(%[
+      aggregate all
+      count_key target
+      pattern1 ok 2\\d\\d
+      pattern2 redirect 3\\d\\d
+      outcast_unmatched true
+    ], 'test.tag2')
+    d3.run do
+      60.times do
+        d3.emit({'target' => '200'})
+        d3.emit({'target' => '300'})
+        d3.emit({'target' => '400'})
+      end
+    end
+    d3.instance.flush_emit(180)
+    emits = d3.emits
+    assert_equal 1, emits.length
+    data = emits[0]
+    assert_equal 'datacount', data[0] # tag
+    assert_equal 60, data[2]['unmatched_count']
+    assert_nil data[2]['unmatched_percentage']
+    assert_equal 60, data[2]['ok_count']
+    assert_equal 50.0, data[2]['ok_percentage']
+    assert_equal 60, data[2]['redirect_count']
+    assert_equal 50.0, data[2]['redirect_percentage']
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datacounter
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-29 00:00:00.000000000 Z
+date: 2012-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -80,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: fluent-plugin-datacounter
-rubygems_version: 1.8.19
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: Output filter plugin to count messages that matches specified conditions