fluent-plugin-numeric-counter 0.1.0 → 0.1.1

data/README.md

@@ -61,6 +61,58 @@ Size specifier (like 10k, 5M, 103g) available as 1024\*\*1, 1024\*\*2, 1024\*\*3
 
 You can try to use negative numbers, and floating point numbers.... (not tested enough).
 
+With 'output\_per\_tag' option and 'tag\_prefix', we get one result message for one tag:
+
+    <match accesslog.{foo,bar}>
+      type numeric_counter
+      unit hour
+      aggregate tag
+      count_key bytes
+      output_per_tag yes
+      tag_prefix byteslog
+      input_tag_remove_prefix accesslog
+      
+      pattern1 SMALL    0 1k
+      pattern2 MIDDLE  1k 1m
+      pattern3 LARGE   1m 10m
+      pattern4 HUGE   10m 1g
+      pattern5 XXXX    1g
+    </match>
+    # => tag: 'byteslog.foo' and 'byteslog.bar'
+    #    message: {'SMALL_count' => 100, ... }
+
+And you can get tested messages count with 'output\_messages' option:
+
+    <match accesslog.{foo,bar}>
+      type numeric_counter
+      unit hour
+      aggregate tag
+      count_key bytes
+      input_tag_remove_prefix accesslog
+      output_messages yes
+      
+      pattern1 SMALL    0 1k
+      pattern2 LARGE   1k
+    </match>
+    # => tag: 'numcount'
+    #    message: {'foo_messages' => xxx, 'bar_messages' => yyy, 'foo_SMALL_count' => 100, ... }
+    
+    <match accesslog.{foo,bar}>
+      type numeric_counter
+      unit hour
+      aggregate tag
+      count_key bytes
+      output_per_tag yes
+      tag_prefix num
+      input_tag_remove_prefix accesslog
+      output_messages yes
+      
+      pattern1 SMALL    0 1k
+      pattern2 LARGE   1k
+    </match>
+    # => tag: 'num.foo' or 'num.bar'
+    #    message: {'messages' => xxx, 'SMALL_count' => 100, ... }
+
 ## TODO
 
 * more tests

data/fluent-plugin-numeric-counter.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-numeric-counter"
-  gem.version       = "0.1.0"
+  gem.version       = "0.1.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{Counts messages, with specified key and numeric value in specified range}
@@ -14,6 +14,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_development_dependency "rake"
   gem.add_development_dependency "fluentd"
   gem.add_runtime_dependency "fluentd"
 end

data/lib/fluent/plugin/out_numeric_counter.rb

@@ -5,11 +5,14 @@ class Fluent::NumericCounterOutput < Fluent::Output
 
   config_param :count_interval, :time, :default => 60
   config_param :unit, :string, :default => nil
+  config_param :output_per_tag, :bool, :default => false
   config_param :aggregate, :string, :default => 'tag'
   config_param :tag, :string, :default => 'numcount'
+  config_param :tag_prefix, :string, :default => nil
   config_param :input_tag_remove_prefix, :string, :default => nil
   config_param :count_key, :string
   config_param :outcast_unmatched, :bool, :default => false
+  config_param :output_messages, :bool, :default => false
 
   # pattern0 reserved as unmatched counts
   config_param :pattern1, :string # string: NAME LOW HIGH
@@ -20,8 +23,6 @@ class Fluent::NumericCounterOutput < Fluent::Output
   
   attr_accessor :counts, :last_checked
 
-  # for test
-  # attr_accessor :count_interval, :unit, :aggregate, :tag, :input_tag_remove_prefix, :count_key, :outcast_unmatched
   attr_accessor :patterns
 
   def parse_num(str)
@@ -77,6 +78,11 @@ class Fluent::NumericCounterOutput < Fluent::Output
       raise Fluent::ConfigError, "unspecified high threshold allowed only in last pattern" if high.nil? and index != @patterns.length - 1
     end
     
+    if @output_per_tag
+      raise Fluent::ConfigError, "tag_prefix must be specified with output_per_tag" unless @tag_prefix
+      @tag_prefix_string = @tag_prefix + '.'
+    end
+
     if @input_tag_remove_prefix
       @removed_prefix_string = @input_tag_remove_prefix + '.'
       @removed_length = @removed_prefix_string.length
@@ -131,53 +137,72 @@ class Fluent::NumericCounterOutput < Fluent::Output
     tag
   end
 
+  def generate_fields(step, target_counts, attr_prefix, output)
+    sum = if @outcast_unmatched
+            target_counts[1..-1].inject(:+)
+          else
+            target_counts.inject(:+)
+          end
+    messages = sum + (@outcast_unmatched ? target_counts[0] : 0)
+
+    target_counts.each_with_index do |count,i|
+      name = @patterns[i][1]
+      output[attr_prefix + name + '_count'] = count
+      output[attr_prefix + name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
+      unless i == 0 and @outcast_unmatched
+        output[attr_prefix + name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
+      end
+      if @output_messages
+        output[attr_prefix + 'messages'] = messages
+      end
+    end
+
+    output    
+  end
+
   def generate_output(counts, step)
+    if @aggregate == :all
+      return generate_fields(step, counts['all'], '', {})
+    end
+
     output = {}
+    counts.keys.each do |tag|
+      generate_fields(step, counts[tag], stripped_tag(tag) + '_', output)
+    end
+    output
+  end
 
+  def generate_output_per_tags(counts, step)
     if @aggregate == :all
-      # index 0 is unmatched
-      sum = if @outcast_unmatched
-              counts['all'][1..-1].inject(:+)
-            else
-              counts['all'].inject(:+)
-            end
-      counts['all'].each_with_index do |count,i|
-        name = @patterns[i][1]
-        output[name + '_count'] = count
-        output[name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
-        unless i == 0 and @outcast_unmatched
-          output[name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
-        end
-      end
-      return output
+      return {'all' => generate_fields(step, counts['all'], '', {})}
     end
 
+    output_pairs = {}
     counts.keys.each do |tag|
-      t = stripped_tag(tag)
-      sum = if @outcast_unmatched
-              counts[tag][1..-1].inject(:+)
-            else
-              counts[tag].inject(:+)
-            end
-      counts[tag].each_with_index do |count,i|
-        name = @patterns[i][1]
-        output[t + '_' + name + '_count'] = count
-        output[t + '_' + name + '_rate'] = ((count * 100.0) / (1.00 * step)).floor / 100.0
-        unless i == 0 and @outcast_unmatched
-          output[t + '_' + name + '_percentage'] = count * 100.0 / (1.00 * sum) if sum > 0
-        end
-      end
+      output_pairs[stripped_tag(tag)] = generate_fields(step, counts[tag], '', {})
     end
-    output
+    output_pairs
   end
 
-  def flush(step)
+  def flush(step) # returns one message
     flushed,@counts = @counts,count_initialized(@counts.keys.dup)
     generate_output(flushed, step)
   end
 
+  def flush_per_tags(step) # returns map of tag - message
+    flushed,@counts = @counts,count_initialized(@counts.keys.dup)
+    generate_output_per_tags(flushed, step)
+  end
+
   def flush_emit(step)
-    Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
+    if @output_per_tag
+      time = Fluent::Engine.now
+      flush_per_tags(step).each do |tag,message|
+        Fluent::Engine.emit(@tag_prefix_string + tag, time, message)
+      end
+    else
+      Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
+    end
   end
 
   def start_watch

data/test/plugin/test_out_numeric_counter.rb

@@ -15,6 +15,19 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     pattern3 u3s 1000000 3000000
   ]
 
+  CONFIG_OUTPUT_PER_TAG = %[
+    count_interval 60
+    aggregate tag
+    output_per_tag true
+    tag_prefix n
+    input_tag_remove_prefix test
+    count_key target
+    pattern1 u100ms 0 100000
+    pattern2 u1s 100000 1000000
+    pattern3 u3s 1000000 3000000
+    output_messages true
+  ]
+
   def create_driver(conf=CONFIG, tag='test')
     Fluent::Test::OutputTestDriver.new(Fluent::NumericCounterOutput, tag).configure(conf)
   end
@@ -42,14 +55,63 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_nil d.instance.input_tag_remove_prefix
     assert_equal false, d.instance.outcast_unmatched
     assert_equal [[0, 'unmatched', nil, nil], [1, 'smallnum', 0.1, 200], [2, 'subnum', -500, -1]], d.instance.patterns
+    assert_equal false, d.instance.output_per_tag
+    assert_equal false, d.instance.output_messages
 
     d = create_driver %[
       count_key key1
       pattern1 x 0.1 10
       pattern2 y 10 11
       pattern3 z 11
+      output_messages yes
     ]
     assert_equal [[0, 'unmatched', nil, nil], [1, 'x', 0.1, 10], [2, 'y', 10, 11], [3, 'z', 11, nil]], d.instance.patterns
+    assert_equal true, d.instance.output_messages
+  end
+
+  def test_configure_output_per_tag
+    d = create_driver %[
+      count_key field1
+      pattern1 smallnum 0.1 200
+      pattern2 subnum   -500 -1
+      output_per_tag yes
+      tag_prefix numcount
+    ]
+
+    assert_equal 60, d.instance.count_interval
+    assert_equal :tag, d.instance.aggregate
+    assert_equal 'numcount', d.instance.tag
+    assert_nil d.instance.input_tag_remove_prefix
+    assert_equal false, d.instance.outcast_unmatched
+    assert_equal [[0, 'unmatched', nil, nil], [1, 'smallnum', 0.1, 200], [2, 'subnum', -500, -1]], d.instance.patterns
+    assert_equal true, d.instance.output_per_tag
+    assert_equal 'numcount', d.instance.tag_prefix
+    assert_equal false, d.instance.output_messages
+
+    d = create_driver %[
+      count_key key1
+      pattern1 x 0.1 10
+      pattern2 y 10 11
+      pattern3 z 11
+      output_per_tag yes
+      tag_prefix n
+      output_messages yes
+    ]
+    assert_equal [[0, 'unmatched', nil, nil], [1, 'x', 0.1, 10], [2, 'y', 10, 11], [3, 'z', 11, nil]], d.instance.patterns
+    assert_equal true, d.instance.output_per_tag
+    assert_equal 'n', d.instance.tag_prefix
+    assert_equal true, d.instance.output_messages
+
+    x_config = %[
+      count_key key1
+      pattern1 x 0.1 10
+      pattern2 y 10 11
+      pattern3 z 11
+      output_per_tag yes
+    ]
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver(x_config)
+    }
   end
 
   def test_countups
@@ -83,6 +145,7 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_equal  120, r1['input_u3s_count']
     assert_equal  2.0, r1['input_u3s_rate']
     assert_equal 20.0, r1['input_u3s_percentage']
+    assert_nil r1['input_messages']
 
     assert_equal    0, r1['input2_unmatched_count']
     assert_equal  0.0, r1['input2_unmatched_rate']
@@ -96,12 +159,20 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_equal    0, r1['input2_u3s_count']
     assert_equal  0.0, r1['input2_u3s_rate']
     assert_equal  0.0, r1['input2_u3s_percentage']
+    assert_nil r1['input2_messages']
+
+    d = create_driver(CONFIG + "\n output_messages yes \n")
+
+    r1 = d.instance.generate_output({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
+    assert_equal 600, r1['input_messages']
+    assert_equal 600, r1['input2_messages']
 
     d = create_driver %[
       aggregate all
       count_key f1
       pattern1 good 1 2
       outcast_unmatched yes
+      output_messages true
     ]
     r2 = d.instance.generate_output({'all' => [60,240]}, 60)
     assert_equal  60, r2['unmatched_count']
@@ -110,6 +181,69 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_equal 240, r2['good_count']
     assert_equal 4.0, r2['good_rate']
     assert_equal 100.0, r2['good_percentage']
+    assert_equal 300, r2['messages']
+  end
+
+  def test_generate_output_per_tag
+    d = create_driver(CONFIG_OUTPUT_PER_TAG + "\n output_messages false \n")
+    # pattern1 u100ms 0 100000
+    # pattern2 u1s 100000 1000000
+    # pattern3 u3s 1000000 3000000
+
+    r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
+    assert_equal 2, r1.keys.size
+
+    r = r1['input']
+    assert_equal   60, r['unmatched_count']
+    assert_equal  1.0, r['unmatched_rate']
+    assert_equal 10.0, r['unmatched_percentage']
+    assert_equal  240, r['u100ms_count']
+    assert_equal  4.0, r['u100ms_rate']
+    assert_equal 40.0, r['u100ms_percentage']
+    assert_equal  180, r['u1s_count']
+    assert_equal  3.0, r['u1s_rate']
+    assert_equal 30.0, r['u1s_percentage']
+    assert_equal  120, r['u3s_count']
+    assert_equal  2.0, r['u3s_rate']
+    assert_equal 20.0, r['u3s_percentage']
+    assert_nil r['messages']
+
+    r = r1['input2']
+    assert_equal    0, r['unmatched_count']
+    assert_equal  0.0, r['unmatched_rate']
+    assert_equal  0.0, r['unmatched_percentage']
+    assert_equal  600, r['u100ms_count']
+    assert_equal 10.0, r['u100ms_rate']
+    assert_equal 100.0, r['u100ms_percentage']
+    assert_equal    0, r['u1s_count']
+    assert_equal  0.0, r['u1s_rate']
+    assert_equal  0.0, r['u1s_percentage']
+    assert_equal    0, r['u3s_count']
+    assert_equal  0.0, r['u3s_rate']
+    assert_equal  0.0, r['u3s_percentage']
+    assert_nil r['messages']
+
+    d = create_driver(CONFIG_OUTPUT_PER_TAG)
+
+    r1 = d.instance.generate_output_per_tags({'test.input' => [60,240,180,120], 'test.input2' => [0,600,0,0]}, 60)
+    assert_equal 600, r1['input']['messages']
+    assert_equal 600, r1['input2']['messages']
+
+    d = create_driver %[
+      aggregate all
+      count_key f1
+      pattern1 good 1 2
+      outcast_unmatched yes
+      output_messages true
+    ]
+    r2 = d.instance.generate_output_per_tags({'all' => [60,240]}, 60)
+    assert_equal  60, r2['all']['unmatched_count']
+    assert_equal 1.0, r2['all']['unmatched_rate']
+    assert_nil r2['all']['unmatched_percentage']
+    assert_equal 240, r2['all']['good_count']
+    assert_equal 4.0, r2['all']['good_rate']
+    assert_equal 100.0, r2['all']['good_percentage']
+    assert_equal 300, r2['all']['messages']
   end
 
   def test_pattern_num
@@ -186,5 +320,93 @@ class NumericCounterOutputTest < Test::Unit::TestCase
     assert_equal 60, r['tag1_unmatched_count']
     assert_equal 1.0, r['tag1_unmatched_rate']
     assert_equal 20, r['tag1_unmatched_percentage']
+
+    d = create_driver(CONFIG, 'test.tag1')
+    d.run do
+      60.times do
+        d.emit({'target' =>  '50000'})
+        d.emit({'target' => '100000'})
+        d.emit({'target' => '100001'})
+        d.emit({'target' => '0.0'})
+        d.emit({'target' => '-1'})
+      end
+    end
+    d.instance.flush_emit(60)
+    emits = d.emits
+    assert_equal 1, emits.length
+    data = emits[0]
+    assert_equal 'numcount', data[0] # tag
+    r = data[2] # message
+    assert_equal 120, r['tag1_u100ms_count']
+    assert_equal 2.0, r['tag1_u100ms_rate']
+    assert_equal 40.0, r['tag1_u100ms_percentage']
+    assert_equal 120, r['tag1_u1s_count']
+    assert_equal 2.0, r['tag1_u1s_rate']
+    assert_equal 40, r['tag1_u1s_percentage']
+    assert_equal 0, r['tag1_u3s_count']
+    assert_equal 0, r['tag1_u3s_rate']
+    assert_equal 0, r['tag1_u3s_percentage']
+    assert_equal 60, r['tag1_unmatched_count']
+    assert_equal 1.0, r['tag1_unmatched_rate']
+    assert_equal 20, r['tag1_unmatched_percentage']
+  end
+
+  def test_emit_output_per_tag
+    d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
+    d.run do
+      60.times do
+        d.emit({'target' =>  '50000'})
+        d.emit({'target' => '100000'})
+        d.emit({'target' => '100001'})
+        d.emit({'target' => '0.0'})
+        d.emit({'target' => '-1'})
+      end
+    end
+    r = d.instance.flush_per_tags(60)
+    assert_equal 1, r.keys.size
+    r1 = r['tag1']
+    assert_equal 120, r1['u100ms_count']
+    assert_equal 2.0, r1['u100ms_rate']
+    assert_equal 40.0, r1['u100ms_percentage']
+    assert_equal 120, r1['u1s_count']
+    assert_equal 2.0, r1['u1s_rate']
+    assert_equal 40, r1['u1s_percentage']
+    assert_equal 0, r1['u3s_count']
+    assert_equal 0, r1['u3s_rate']
+    assert_equal 0, r1['u3s_percentage']
+    assert_equal 60, r1['unmatched_count']
+    assert_equal 1.0, r1['unmatched_rate']
+    assert_equal 20, r1['unmatched_percentage']
+    assert_equal 300, r1['messages']
+
+    d = create_driver(CONFIG_OUTPUT_PER_TAG, 'test.tag1')
+    d.run do
+      60.times do
+        d.emit({'target' =>  '50000'})
+        d.emit({'target' => '100000'})
+        d.emit({'target' => '100001'})
+        d.emit({'target' => '0.0'})
+        d.emit({'target' => '-1'})
+      end
+    end
+    d.instance.flush_emit(60)
+    emits = d.emits
+    assert_equal 1, emits.length
+    data = emits[0]
+    assert_equal 'n.tag1', data[0] # tag
+    r = data[2] # message
+    assert_equal 120, r['u100ms_count']
+    assert_equal 2.0, r['u100ms_rate']
+    assert_equal 40.0, r['u100ms_percentage']
+    assert_equal 120, r['u1s_count']
+    assert_equal 2.0, r['u1s_rate']
+    assert_equal 40, r['u1s_percentage']
+    assert_equal 0, r['u3s_count']
+    assert_equal 0, r['u3s_rate']
+    assert_equal 0, r['u3s_percentage']
+    assert_equal 60, r['unmatched_count']
+    assert_equal 1.0, r['unmatched_rate']
+    assert_equal 20, r['unmatched_percentage']
+    assert_equal 300, r['messages']
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-numeric-counter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-06 00:00:00.000000000 Z
+date: 2012-08-08 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement