fluent-plugin-numeric-monitor 0.1.4 → 0.1.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f0e34e02fb3c60ee05565ff9e7ed9c31bbf1f677
+  data.tar.gz: b74595ed3a9194bb3da3f21d8d47749b97059bcd
+SHA512:
+  metadata.gz: 9031f88688412a012a965d9240639e51c65716bcc5248085c8bdff1e85bcde52b8b12d3703df11442464427a5b9c1459c26d8bfec907a60ec18fdfe62d72e21f
+  data.tar.gz: 9ee81f7dfbe2483631df14dbea7553cced143f62c5631d23c25386cc1dcd6c669aa7b92fd59870b23a1c380090293031d44a9c6a9ee9c68ca2e71c22c8d8e31b

data/README.md

@@ -26,6 +26,54 @@ Output messages like:
 
     {"min":3012,"max":913243,"avg":100123.51,"percentile_90":154390,"percentile_95":223110,"num":50012}
 
+
+## Parameters
+
+* monitor\_key (required)
+
+    The key to monitor in the event record.
+    
+* percentiles
+
+    Activate the percentile monitoring. Must be specified between `1` and `99` by integer separeted by , (comma). 
+
+* tag
+
+    The output tag. Default is `monitor`. 
+
+* tag\_prefix
+
+    The prefix string which will be added to the input tag. `output_per_tag yes` must be specified together. 
+    
+* input\_tag\_remove\_prefix
+
+    The prefix string which will be removed from the input tag. 
+
+* count\_interval
+
+    The interval time to monitor in seconds. Default is `60`. 
+    
+* unit
+
+    The interval time to monitor specified an unit (either of `minute`, `hour`, or `day`). 
+    Use either of `count_interval` or `unit`.
+    
+* aggregate
+
+    Calculate in each input `tag` separetely, or `all` records in a mass. Default is `tag`
+    
+* ouput\_per\_tag
+
+    Emit for each input tag. `tag_prefix` must be specified together. Default is `no`. 
+
+* output\_key\_prefix
+
+    The prefix string which will be added to the output key. 
+    
+* samples\_limit
+
+    The limit number of sampling. Default is `1000000`. 
+
 ## TODO
 
 * more tests

data/fluent-plugin-numeric-monitor.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-numeric-monitor"
-  gem.version       = "0.1.4"
+  gem.version       = "0.1.5"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{Fluentd plugin to calculate min/max/avg/Xpercentile values, and emit these data as message}

data/lib/fluent/plugin/out_numeric_monitor.rb

@@ -6,7 +6,9 @@ class Fluent::NumericMonitorOutput < Fluent::Output
   config_param :count_interval, :time, :default => 60
   config_param :unit, :string, :default => nil
   config_param :tag, :string, :default => 'monitor'
+  config_param :tag_prefix, :string, :default => nil
 
+  config_param :output_per_tag, :bool, :default => false
   config_param :aggregate, :default => 'tag' do |val|
     case val
     when 'tag' then :tag
@@ -17,6 +19,7 @@ class Fluent::NumericMonitorOutput < Fluent::Output
   end
   config_param :input_tag_remove_prefix, :string, :default => nil
   config_param :monitor_key, :string
+  config_param :output_key_prefix, :string, :default => nil
   config_param :percentiles, :default => nil do |val|
     values = val.split(",").map(&:to_i)
     if values.select{|i| i < 1 or i > 99 }.size > 0
@@ -46,6 +49,16 @@ class Fluent::NumericMonitorOutput < Fluent::Output
       @removed_prefix_string = @input_tag_remove_prefix + '.'
       @removed_length = @removed_prefix_string.length
     end
+
+    @key_prefix_string = ''
+    if @output_key_prefix
+      @key_prefix_string = @output_key_prefix + '_'
+    end
+
+    if (@output_per_tag || @tag_prefix) && (!@output_per_tag || !@tag_prefix)
+      raise Fluent::ConfigError, 'Specify both of output_per_tag and tag_prefix'
+    end
+    @tag_prefix_string = @tag_prefix + '.' if @output_per_tag
     
     @count = count_initialized
     @mutex = Mutex.new
@@ -106,42 +119,46 @@ class Fluent::NumericMonitorOutput < Fluent::Output
     tag
   end
 
-  def generate_output(count)
-    output = {}
-    if @aggregate == :all
-      c = count['all']
-      if c[:num] then output['num'] = c[:num] end
-      if c[:min] then output['min'] = c[:min] end
-      if c[:max] then output['max'] = c[:max] end
-      if c[:num] > 0 then output['avg'] = (c[:sum] * 100.0 / (c[:num] * 1.0)).round / 100.0 end
-      if @percentiles
-        sorted = c[:sample].sort
-        @percentiles.each do |p|
-          i = (c[:num] * p / 100).floor
-          if i > 0
-            i -= 1
-          end
-          output["percentile_#{p}"] = sorted[i]
+  def generate_fields(count, key_prefix = '', output = {})
+    output[key_prefix + 'num'] = count[:num] if count[:num]
+    output[key_prefix + 'min'] = count[:min] if count[:min]
+    output[key_prefix + 'max'] = count[:max] if count[:max]
+    output[key_prefix + 'avg'] = (count[:sum] / (count[:num] * 1.0)) if count[:num] > 0
+    if @percentiles
+      sorted = count[:sample].sort
+      @percentiles.each do |p|
+        i = (count[:num] * p / 100).floor
+        if i > 0
+          i -= 1
         end
+        output[key_prefix + "percentile_#{p}"] = sorted[i]
       end
-      return output
     end
+    output
+  end
 
-    count.keys.each do |tag|
-      t = stripped_tag(tag)
-      c = count[tag]
-      if c[:num] then output[t + '_num'] = c[:num] end
-      if c[:min] then output[t + '_min'] = c[:min] end
-      if c[:max] then output[t + '_max'] = c[:max] end
-      if c[:num] > 0 then output[t + '_avg'] = (c[:sum] * 100.0 / (c[:num] * 1.0)).round / 100.0 end
-      if @percentiles
-        sorted = c[:sample].sort
-        @percentiles.each do |p|
-          i = (c[:num] * p / 100).floor
-          if i > 0
-            i -= 1
-          end
-          output[t + "_percentile_#{p}"] = sorted[i]
+  def generate_output(count)
+    if @aggregate == :all
+      if @output_per_tag
+        # tag_prefix_all: { 'key_prefix_min' => -10, 'key_prefix_max' => 10, ... } }
+        output = {'all' => generate_fields(count['all'], @key_prefix_string)}
+      else
+        # tag: { 'key_prefix_min' => -10, 'key_prefix_max' => 10, ... }
+        output = generate_fields(count['all'], @key_prefix_string)
+      end
+    else
+      output = {}
+      if @output_per_tag
+        # tag_prefix_tag1: { 'key_prefix_min' => -10, 'key_prefix_max' => 10, ... }
+        # tag_prefix_tag2: { 'key_prefix_min' => -10, 'key_prefix_max' => 10, ... }
+        count.keys.each do |tag|
+          output[stripped_tag(tag)] = generate_fields(count[tag], @key_prefix_string)
+        end
+      else
+        # tag: { 'key_prefix_tag1_min' => -10, 'key_prefix_tag1_max' => 10, ..., 'key_prefix_tag2_min' => -10, 'key_prefix_tag2_max' => 10, ... }
+        count.keys.each do |tag|
+          key_prefix = @key_prefix_string + stripped_tag(tag) + '_'
+          generate_fields(count[tag], key_prefix, output)
         end
       end
     end
@@ -154,7 +171,14 @@ class Fluent::NumericMonitorOutput < Fluent::Output
   end
 
   def flush_emit
-    Fluent::Engine.emit(@tag, Fluent::Engine.now, flush)
+    if @output_per_tag
+      time = Fluent::Engine.now
+      flush.each do |tag, message|
+        Fluent::Engine.emit(@tag_prefix_string + tag, time, message)
+      end
+    else
+      Fluent::Engine.emit(@tag, Fluent::Engine.now, flush)
+    end
   end
 
   def countups(tag, min, max, sum, num, sample)

data/test/plugin/test_out_numeric_monitor.rb

@@ -18,6 +18,19 @@ class NumericMonitorOutputTest < Test::Unit::TestCase
   end
 
   def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver CONFIG + %[
+        output_per_tag true
+      ]
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver CONFIG + %[
+        tag_prefix prefix
+      ]
+    }
     #TODO
   end
 
@@ -115,4 +128,228 @@ class NumericMonitorOutputTest < Test::Unit::TestCase
     assert_equal 2, r['test_avg']
     assert_equal 3, r['test_num']
   end
+
+  def test_output_per_tag
+    d = create_driver(CONFIG + %[
+      aggregate tag
+      output_per_tag true
+      tag_prefix tag_prefix
+    ], 'tag')
+    d.run do
+      d.tag = 'tag1'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+      d.tag = 'tag2'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+    end
+    d.instance.flush_emit
+    assert_equal 2, d.emits.size
+    tag, r = d.emits[0][0], d.emits[0][2]
+    assert_equal 'tag_prefix.tag1', tag
+    assert_equal 1, r['min']
+    assert_equal 3, r['max']
+    assert_equal 2, r['avg']
+    assert_equal 3, r['num']
+    tag, r = d.emits[1][0], d.emits[1][2]
+    assert_equal 'tag_prefix.tag2', tag
+    assert_equal 1, r['min']
+    assert_equal 3, r['max']
+    assert_equal 2, r['avg']
+    assert_equal 3, r['num']
+
+    d = create_driver(CONFIG + %[
+      aggregate tag
+      output_per_tag false
+      tag output_tag
+    ], 'tag')
+    d.run do
+      d.tag = 'tag1'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+      d.tag = 'tag2'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+    end
+    d.instance.flush_emit
+    assert_equal 1, d.emits.size
+    tag, r = d.emits[0][0], d.emits[0][2]
+    assert_equal 'output_tag', tag
+    assert_equal 1, r['tag1_min']
+    assert_equal 3, r['tag1_max']
+    assert_equal 2, r['tag1_avg']
+    assert_equal 3, r['tag1_num']
+    assert_equal 1, r['tag2_min']
+    assert_equal 3, r['tag2_max']
+    assert_equal 2, r['tag2_avg']
+    assert_equal 3, r['tag2_num']
+
+    d = create_driver(CONFIG + %[
+      aggregate all
+      output_per_tag true
+      tag_prefix tag_prefix
+    ], 'tag')
+    d.run do
+      d.tag = 'tag1'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+      d.tag = 'tag2'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+    end
+    d.instance.flush_emit
+    assert_equal 1, d.emits.size
+    tag = d.emits[0][0]
+    r = d.emits[0][2]
+    assert_equal 'tag_prefix.all', tag
+    assert_equal 1, r['min']
+    assert_equal 3, r['max']
+    assert_equal 2, r['avg']
+    assert_equal 6, r['num']
+
+    d = create_driver(CONFIG + %[
+      aggregate all
+      output_per_tag false
+      tag output_tag
+    ], 'tag')
+    d.run do
+      d.tag = 'tag1'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+      d.tag = 'tag2'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+    end
+    d.instance.flush_emit
+    assert_equal 1, d.emits.size
+    tag = d.emits[0][0]
+    r = d.emits[0][2]
+    assert_equal 'output_tag', tag
+    assert_equal 1, r['min']
+    assert_equal 3, r['max']
+    assert_equal 2, r['avg']
+    assert_equal 6, r['num']
+  end
+
+  def test_output_key_prefix
+    d = create_driver(CONFIG + %[
+      aggregate tag
+      output_per_tag true
+      tag_prefix tag_prefix
+      output_key_prefix key_prefix
+    ], 'tag')
+    d.run do
+      d.tag = 'tag1'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+      d.tag = 'tag2'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+    end
+    d.instance.flush_emit
+    assert_equal 2, d.emits.size
+    tag, r = d.emits[0][0], d.emits[0][2]
+    assert_equal 'tag_prefix.tag1', tag
+    assert_equal 1, r['key_prefix_min']
+    assert_equal 3, r['key_prefix_max']
+    assert_equal 2, r['key_prefix_avg']
+    assert_equal 3, r['key_prefix_num']
+    tag, r = d.emits[1][0], d.emits[1][2]
+    assert_equal 'tag_prefix.tag2', tag
+    assert_equal 1, r['key_prefix_min']
+    assert_equal 3, r['key_prefix_max']
+    assert_equal 2, r['key_prefix_avg']
+    assert_equal 3, r['key_prefix_num']
+
+    d = create_driver(CONFIG + %[
+      aggregate tag
+      output_per_tag false
+      tag output_tag
+      output_key_prefix key_prefix
+    ], 'tag')
+    d.run do
+      d.tag = 'tag1'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+      d.tag = 'tag2'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+    end
+    d.instance.flush_emit
+    assert_equal 1, d.emits.size
+    tag, r = d.emits[0][0], d.emits[0][2]
+    assert_equal 'output_tag', tag
+    assert_equal 1, r['key_prefix_tag1_min']
+    assert_equal 3, r['key_prefix_tag1_max']
+    assert_equal 2, r['key_prefix_tag1_avg']
+    assert_equal 3, r['key_prefix_tag1_num']
+    assert_equal 1, r['key_prefix_tag2_min']
+    assert_equal 3, r['key_prefix_tag2_max']
+    assert_equal 2, r['key_prefix_tag2_avg']
+    assert_equal 3, r['key_prefix_tag2_num']
+
+    d = create_driver(CONFIG + %[
+      aggregate all
+      output_per_tag true
+      tag_prefix tag_prefix
+      output_key_prefix key_prefix
+    ], 'tag')
+    d.run do
+      d.tag = 'tag1'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+      d.tag = 'tag2'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+    end
+    d.instance.flush_emit
+    assert_equal 1, d.emits.size
+    tag = d.emits[0][0]
+    r = d.emits[0][2]
+    assert_equal 'tag_prefix.all', tag
+    assert_equal 1, r['key_prefix_min']
+    assert_equal 3, r['key_prefix_max']
+    assert_equal 2, r['key_prefix_avg']
+    assert_equal 6, r['key_prefix_num']
+
+    d = create_driver(CONFIG + %[
+      aggregate all
+      output_per_tag false
+      tag output_tag
+      output_key_prefix key_prefix
+    ], 'tag')
+    d.run do
+      d.tag = 'tag1'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+      d.tag = 'tag2'
+      d.emit({'field1' => 1})
+      d.emit({'field1' => 2})
+      d.emit({'field1' => 3})
+    end
+    d.instance.flush_emit
+    assert_equal 1, d.emits.size
+    tag = d.emits[0][0]
+    r = d.emits[0][2]
+    assert_equal 'output_tag', tag
+    assert_equal 1, r['key_prefix_min']
+    assert_equal 3, r['key_prefix_max']
+    assert_equal 2, r['key_prefix_avg']
+    assert_equal 6, r['key_prefix_num']
+  end
 end

metadata

@@ -1,46 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-numeric-monitor
 version: !ruby/object:Gem::Version
-  version: 0.1.4
-  prerelease: 
+  version: 0.1.5
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-08 00:00:00.000000000 Z
+date: 2013-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Fluentd plugin to calculate min/max/avg/Xpercentile values, and emit
@@ -62,27 +57,26 @@ files:
 - test/plugin/test_out_numeric_monitor.rb
 homepage: https://github.com/tagomoris/fluent-plugin-numeric-monitor
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fluentd plugin to calculate min/max/avg/Xpercentile values
 test_files:
 - test/helper.rb