fluent-plugin-elapsed-time 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e6720e8ded7541bd64fcf697c652f26dc8698d06
-  data.tar.gz: b9fc406ba89a260a898d2f0eebbae70f0494f553
+  metadata.gz: d14cf6319e26776194fbb0919e67088cd7ea95b4
+  data.tar.gz: 3ddedf3e28e320fafcff40ceab676373afcbaa57
 SHA512:
-  metadata.gz: 6b0eb1286b7aa683a562f9b9fb61b78176453543dc1306fe40823cde5dcea4bcdff52239059782dd5e4ad3d9a375640660a0f81254bdd842a789a565c37fddb0
-  data.tar.gz: df7c63be74f4ea39658f73eb2cab8035ce3b26851b2dea9adf357fd0b7575d2a3a17ed3fc2bb33647af5b381bed749a19e780c14079ca2382cc8f8f9b226ed4e
+  metadata.gz: a2cdc21c833167dfdbcb966eb94985dc368f55887bda1b19a28711b400fdfe3a840b06bf654688f1337b8baa9f74fb8bff5c960797c7365232a5e30a3afead4d
+  data.tar.gz: 851810c1aa907d6be79c9f54fb7cd1b41747006553979f363659b51cbf8b17162b039e700cc0f2af574a186ec71c1a49b202a2e4f4a99772e3e409d76dcc5d3c

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.0.3  (2014/01/30)
+
+Enhancement:
+
+* Add `aggregate` option
+* Add `add_tag_prefix` and `remove_tag_prefix` options
+* Add `remove_tag_slice` option
+
 ## 0.0.2  (2014/01/23)
 
 Changes:

data/README.md

@@ -21,6 +21,7 @@ Following example measures the max and average time taken to process [fluent-plu
   type elapsed_time
   tag elapsed
   interval 60
+  each message
   <store>
     type grep
     exclude foobar
@@ -45,11 +46,15 @@ Following example measures the max and average time taken to process [fluent-plu
 </match>
 ```
 
-## Option Parameters
+Output will be like
 
-* tag
+```
+elapsed: {"max":1.011,"avg":0.002","num":10}
+```
 
-    The output tag name. Default is `elapsed`
+where `max` and `avg` are the maximum and average elapsed times, and `num` is the number of messages.
+
+## Option Parameters
 
 * interval
 
@@ -59,6 +64,35 @@ Following example measures the max and average time taken to process [fluent-plu
 
     Measure time for each `message` or `es` (event stream). Please notice that the event stream (would be a msgpack) will be unpacked if `message` is specified, which would cause performance degradation. Default is `es`.
 
+* tag
+
+    The output tag name. Default is `elapsed`
+
+* add_tag_prefix
+
+    Add tag prefix for output message
+
+* remove_tag_prefix
+
+    Remove tag prefix for output message
+
+* remove_tag_slice *min..max*
+
+    Remove tag parts by slice function. FYI: This option behaves like `tag.split('.').slice(min..max)`.
+
+    For example,
+
+        remove_tag_slice 0..-2
+
+    changes an input tag `foo.bar.host1` to `foo.bar`. 
+
+* aggregate
+
+    Measure and emit outputs for each `tag` or `all`. Default is `all`.
+
+    `all` measures `max` and `avg` for all input messages.
+    `tag` measures `max` and `avg` for each tag *modified* by `add_tag_prefix`, `remove_tag_prefix`, or `remove_tag_slice`. 
+
 ## ChangeLog
 
 See [CHANGELOG.md](CHANGELOG.md) for details.

data/fluent-plugin-elapsed-time.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-elapsed-time"
-  gem.version     = "0.0.2"
+  gem.version     = "0.0.3"
   gem.authors     = ["Naotoshi Seo"]
   gem.email       = "sonots@gmail.com"
   gem.homepage    = "https://github.com/sonots/fluent-plugin-elapsed-time"

data/lib/fluent/plugin/out_elapsed_time.rb

@@ -3,6 +3,10 @@ module Fluent
     Plugin.register_output('elapsed_time', self)
 
     config_param :tag, :string, :default => 'elapsed'
+    config_param :add_tag_prefix, :string, :default => nil
+    config_param :remove_tag_prefix, :string, :default => nil
+    config_param :remove_tag_slice, :string, :default => nil
+    config_param :aggregate, :string, :default => 'all'
     config_param :interval, :time, :default => 60
     config_param :each, :default => :es do |val|
       case val.downcase
@@ -18,10 +22,15 @@ module Fluent
     def initialize
       super
       @outputs = []
-      @elapsed = []
+      @elapsed = {}
     end
 
-    attr_reader :outputs, :elapsed
+    # for test
+    attr_reader :outputs
+
+    def elapsed(tag = :all)
+      @elapsed[tag] ||= []
+    end
 
     def configure(conf)
       super
@@ -39,6 +48,50 @@ module Fluent
         @outputs << output
       }
 
+      case @aggregate
+      when 'all'
+        raise ConfigError, "out_elapsed_time: `tag` must be specified with aggregate all" if @tag.nil?
+      when 'tag'
+        raise ConfigError, "out_elapsed_time: `add_tag_prefix` or `remove_tag_prefix` must be specified with aggregate tag" if @add_tag_prefix.nil? and @remove_tag_prefix.nil?
+      else
+        raise ConfigError, "out_elapsed_time: aggregate allows `tag` or `all`"
+      end
+
+      @tag_slice_proc =
+        if @remove_tag_slice
+          lindex, rindex = @remove_tag_slice.split('..', 2)
+          if lindex.nil? or rindex.nil? or lindex !~ /^-?\d+$/ or rindex !~ /^-?\d+$/
+            raise Fluent::ConfigError, "out_elapsed_time: remove_tag_slice must be formatted like [num]..[num]"
+          end
+          l, r = lindex.to_i, rindex.to_i
+          Proc.new {|tag| (tags = tag.split('.')[l..r]).nil? ? "" : tags.join('.') }
+        else
+          Proc.new {|tag| tag }
+        end
+
+      @tag_prefix = "#{@add_tag_prefix}." if @add_tag_prefix
+      @tag_prefix_match = "#{@remove_tag_prefix}." if @remove_tag_prefix
+      @tag_proc =
+        if @tag_prefix and @tag_prefix_match
+          Proc.new {|tag| "#{@tag_prefix}#{lstrip(@tag_slice_proc.call(tag), @tag_prefix_match)}" }
+        elsif @tag_prefix_match
+          Proc.new {|tag| lstrip(@tag_slice_proc.call(tag), @tag_prefix_match) }
+        elsif @tag_prefix
+          Proc.new {|tag| "#{@tag_prefix}#{@tag_slice_proc.call(tag)}" }
+        elsif @tag
+          Proc.new {|tag| @tag }
+        else
+          Proc.new {|tag| @tag_slice_proc.call(tag) }
+        end
+
+      @push_elapsed_proc =
+        case @aggregate
+        when 'all'
+          Proc.new {|tag, elapsed_time| elapsed(:all) << elapsed_time }
+        when 'tag'
+          Proc.new {|tag, elapsed_time| elapsed(tag) << elapsed_time }
+        end
+
       @emit_proc =
         if @each == :message
           chain = NullOutputChain.instance
@@ -47,7 +100,8 @@ module Fluent
             es.each do |time, record|
               @outputs.each {|output| output.emit(tag, OneEventStream.new(time, record), chain) }
               finish = Time.now
-              @elapsed << (finish - start).to_f
+              elapsed = (finish - start).to_f
+              @push_elapsed_proc.call(@tag_proc.call(tag), elapsed)
               start = finish
             end
           }
@@ -56,7 +110,8 @@ module Fluent
           Proc.new {|tag, es|
             t = Time.now
             @outputs.each {|output| output.emit(tag, es, chain) }
-            @elapsed << (Time.now - t).to_f
+            elapsed = (Time.now - t).to_f
+            @push_elapsed_proc.call(@tag_proc.call(tag), elapsed)
           }
         end
     end
@@ -88,12 +143,14 @@ module Fluent
     end
 
     def flush_emit
-      elapsed, @elapsed = @elapsed, []
-      unless elapsed.empty?
-        max = elapsed.max
-        num = elapsed.size
-        avg = elapsed.map(&:to_f).inject(:+) / num.to_f
-        Engine.emit(@tag, Engine.now, {"max" => max, "avg" => avg, "num" => num})
+      elapseds, @elapsed = @elapsed, {}
+      elapseds.each do |tag, elapsed|
+        unless elapsed.empty?
+          max = elapsed.max
+          num = elapsed.size
+          avg = elapsed.map(&:to_f).inject(:+) / num.to_f
+          Engine.emit(tag, Engine.now, {"max" => max, "avg" => avg, "num" => num})
+        end
       end
     end
 
@@ -101,5 +158,9 @@ module Fluent
       @emit_proc.call(tag, es)
       chain.next
     end
+
+    def lstrip(string, substring)
+      string.index(substring) == 0 ? string[substring.size..-1] : string
+    end
   end
 end

data/spec/out_elapsed_time_spec.rb

@@ -86,5 +86,40 @@ describe Fluent::ElapsedTimeOutput do
         driver.instance.elapsed.size.should == 4
       }
     end
+
+    context 'each message with aggregate tag' do
+      let(:config) { CONFIG + %[each message\naggregate tag\nadd_tag_prefix elapsed]}
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+      end
+      it {
+        driver.run { messages.each {|message| driver.emit({'message' => message}, time) } }
+        driver.instance.elapsed("elapsed.#{tag}").size.should == 4
+      }
+    end
+
+    context 'each es with aggregate tag' do
+      let(:config) { CONFIG + %[each es\naggregate tag\nadd_tag_prefix elapsed]}
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+      end
+      it {
+        driver.run { messages.each {|message| driver.emit({'message' => message}, time) } }
+        driver.instance.elapsed("elapsed.#{tag}").size.should == 4
+        driver.instance.flush_emit
+      }
+    end
+
+    context 'remove_tag_slice' do
+      let(:config) { CONFIG + %[remove_tag_slice 0..-2\naggregate tag\nadd_tag_prefix elapsed]}
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+      end
+      let(:expected_tag) { tag.split('.')[0..-2].join('.') }
+      it {
+        driver.run { messages.each {|message| driver.emit({'message' => message}, time) } }
+        driver.instance.elapsed("elapsed.#{expected_tag}").size.should == 4
+      }
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elapsed-time
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-23 00:00:00.000000000 Z
+date: 2014-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd