fluent-plugin-measure_time 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 193e1f487b71b8b3e05144ada9ef36651b95d230
+  data.tar.gz: d0a5363194ca0c40b26d242dff97d703e7850c11
+SHA512:
+  metadata.gz: c76cda2d36d046e9496509fe3f139394660b1e2791294489bcfbb1c78706af03830886e972f081ef6c07a77512d059a12b7a405ddfd64e49c35f93eeb204b038
+  data.tar.gz: cbdd31735b323db8d30d5292b66cf939131f491b07968312f00691684859310722d497c3cc20ae55c87fc3012414e2c672db4e1d4eb70b1d1ec4650bf49a2d1c

data/.gitignore

@@ -0,0 +1,15 @@
+/*.gem
+~*
+#*
+*~
+.bundle
+Gemfile.lock
+.rbenv-version
+vendor
+doc/*
+tmp/*
+coverage
+.yardoc
+pkg/
+.ruby-version
+*.log

data/.rspec

@@ -0,0 +1,2 @@
+--colour
+--format documentation

data/.travis.yml

@@ -0,0 +1,6 @@
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+gemfile:
+  - Gemfile

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0  (2014/04/12)
+
+First release

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Naotoshi SEO
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,108 @@
+# fluent-plugin-measure_time
+
+[![Build Status](https://secure.travis-ci.org/sonots/fluent-plugin-measure_time.png?branch=master)](http://travis-ci.org/sonots/fluent-plugin-measure_time)
+[![Code Climate](https://codeclimate.com/github/sonots/fluent-plugin-measure_time.png)](https://codeclimate.com/github/sonots/fluent-plugin-measure_time)
+
+Fluentd plugin to measure elapsed time to process messages
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-measure_time
+
+## Configuration
+
+This plugin is doing something tricky, which extends arbitrary plugins so that it can use `<measure_time></measure_time>` directive to measure elapsed times. 
+
+Example:
+
+```apache
+<source>
+  type measure_time
+  # This makes available the `masure_time` directive for all plugins
+</source>
+
+<source>
+  type forward
+  port 24224
+  <measure_time>
+    tag measure_time
+    hook on_message
+  </measure_time>
+</source>
+
+<match measure_time>
+  type stdout
+</match>
+```
+
+This example hooks the [on_message](https://github.com/fluent/fluentd/blob/e5a9a4ca03d18b45fdb89061d8251592a044e9fc/lib/fluent/plugin/in_forward.rb#L112) method of in_forward plugin, and measures how long it takes for processing. Output becomes as below:
+
+```
+measure_time: {"time":0.000849735,"class":"Fluent::ForwardInput","hook":"on_message","object_id":83935080}
+```
+
+where `time` denotes the measured elapsed time, and `class`, `hook`, and `object_id` denotes the hooked class, the hooked method, and the object id of the plugin instance. 
+
+Example: interval
+
+With `interval` option, this plugin compute statistics of measured elapsed times in each interval
+
+```apache
+<source>
+  type measure_time
+</source>
+
+<source>
+  type forward
+  port 24224
+  <measure_time>
+    tag measure_time
+    interval 60
+    hook on_message
+  </measure_time>
+</source>
+
+<match measure_time>
+  type stdout
+</match>
+```
+
+Output becomes as below:
+
+```
+measure_time: {"max":1.011,"avg":0.002","num":10,"class":"Fluent::ForwardInput","hook":"on_message","object_id":83935080}
+```
+
+where `max` and `avg` are the maximum and average elapsed times, and `num` is the number of being called in each interval.
+
+## Parameters
+
+* tag
+
+    The output tag name. Default is `measure_time`
+
+* hook (required)
+
+    Specify the method to measure time.
+
+* interval
+
+    The time interval to emit measurement results. Default is nil which do not compute statistics and emit the time in each measurement.
+    
+## ChangeLog
+
+See [CHANGELOG.md](CHANGELOG.md) for details.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Copyright
+
+Copyright (c) 2014 Naotoshi Seo. See [LICENSE](LICENSE) for details.

data/Rakefile

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+task :default => :spec
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+    sh 'irb -rubygems -I lib'
+end
+task :c => :console

data/benchmark/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gem 'fluentd'
+gem 'dummer'
+gem 'fluent-plugin-keep-forward'
+gem 'fluent-plugin-flowcounter-simple'
+gem 'fluent-plugin-measure_time', path: '../'

data/benchmark/README.md

@@ -0,0 +1,93 @@
+# Fluentd benchmark - forward
+
+This benchmarks following architecture scenario:
+
+```
+  Agent Node                                     Receiver Node
+  +-----------------------------------+          +-----------------+
+  | +-----------+      +-----------+  |          |  +-----------+  |
+  | |           |      |           |  |          |  |           |  |
+  | | Log File  +----->|  Fluentd  +--------------->|  Fluentd  |  |
+  | |           |      |           |  |          |  |           |  |
+  | +-----------+  in_tail ----- out_forward   in_forward  -----+  |
+  +-----------------------------------+          +-----------------+
+```
+
+## Setup Fluentd Receiver
+
+Assum ruby is installed
+
+```
+git clone https://github.com/sonots/fluentd-benchmark
+cd fluentd-benchmark/one_forward
+bundle
+bundle exec fluentd -c receiver.conf
+```
+
+## Setup Fluentd Agent
+
+Assume ruby is installed
+
+```
+git clone https://github.com/sonots/fluentd-benchmark
+cd fluentd-benchmark/one_forward
+bundle
+bundle exec fluentd -c agent.conf
+```
+
+## Run benchmark tool and measure
+
+Run at Fluentd agent server. 
+
+This tool generates a log file to dummy.log and Fluentd agent will read and send data to receiver. 
+
+```
+cd fluentd-benchmark/one_forward
+bundle exec dummer -c dummer.conf
+```
+
+You may increase the rate (messages/sec) of generating log by -r option to benchmark. 
+
+```
+bundle exec dummer -c dummer.conf -r 100000
+```
+
+You should see an output on Fluentd receiver as following. This will tell you the performance of fluentd processing. 
+
+```
+2014-02-20 17:20:55 +0900 [info]: plugin:out_flowcounter_simple count:500       indicator:num   unit:second
+2014-02-20 17:20:56 +0900 [info]: plugin:out_flowcounter_simple count:500       indicator:num   unit:second
+2014-02-20 17:20:57 +0900 [info]: plugin:out_flowcounter_simple count:500       indicator:num   unit:second
+```
+
+You may use `iostat -dkxt 1`, `vmstat 1`, `top -c`, `free`, or `dstat` commands to measure system resources. 
+
+## Sample Result
+
+This is a sample result running on my environement
+
+
+Machine Spec
+
+```
+CPU	Xeon E5-2670 2.60GHz x 2 (32 Cores)
+Memory	24G
+Disk	300G(10000rpm) x 2 [SAS-HDD]
+OS CentOS release 6.2 (Final)
+```
+
+Result
+
+
+| rate of writing (lines/sec) | reading (lines/sec)   | CPU (%) | Memory (kB) | Remarks |
+|-----------------------------|-----------------------|---------|-------------|---------|
+| 10                          | 10                    | 0.2     | 29304       |         |
+| 100                         | 100                   | 0.3     | 35812       |         |
+| 1000                        | 1000                  | 1.3     | 37864       |         |
+| 10000                       | 10000                 | 6.6     | 39912       |         |
+| 100000                      | 100000                | 62      | 39912       |         |
+| 200000                      | 157148                | 100.4   | 36280       | MAX     |
+| 300000                      | N/A                   |         |             |         |
+| 400000                      | N/A                   |         |             |         |
+| 5247047                     | N/A                   |         |             | MAX of dummer tool        |
+

data/benchmark/agent.conf

@@ -0,0 +1,24 @@
+<source>
+  type tail
+  path dummy.log
+  pos_file /var/tmp/_var_log_dummy.pos
+  format none
+  tag dummy
+</source>
+<match dummy>
+  type copy
+  <store>
+    type keep_forward
+    flush_interval 0
+    try_flush_interval 1
+    buffer_chunk_limit 1m
+    buffer_queue_limit 64
+    num_threads 3
+    keepforward thread
+    keepalive true
+    <server>
+      host 127.0.0.1 # FIX ME
+      port 24224
+    </server>
+  </store>
+</match>

data/benchmark/dummer.conf

@@ -0,0 +1,4 @@
+configure 'sample' do
+  output "dummy.log"
+  message "time:2013-11-25 00:23:52 +0900\tlevel:ERROR\tmethod:POST\turi:/api/v1/people\treqtime:3.1983877060667103"
+end

data/benchmark/hooked_in_forward.conf

@@ -0,0 +1,19 @@
+<source>
+  type measure_time
+</source>
+
+<source>
+  type forward
+  port 24224
+  <measure_time>
+    hook on_message
+  </measure_time>
+</source>
+<match dummy>
+  type flowcounter_simple
+  unit second
+</match>
+
+<match measure_time>
+  type stdout
+</match>

data/benchmark/in_forward.conf

@@ -0,0 +1,8 @@
+<source>
+  type forward
+  port 24224
+</source>
+<match dummy>
+  type flowcounter_simple
+  unit second
+</match>

data/benchmark/patched_in_forward.conf

@@ -0,0 +1,16 @@
+# bundle exec fluentd -c patched_in_forward.conf -p plugin
+<source>
+  type forward
+  port 24224
+  <elapsed>
+    hook on_message
+  </elapsed>
+</source>
+<match dummy>
+  type flowcounter_simple
+  unit second
+</match>
+
+<match elapsed>
+  type stdout
+</match>

data/benchmark/plugin/in_forward.rb

@@ -0,0 +1,345 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+  class ForwardInput < Input
+    Plugin.register_input('forward', self)
+
+    def initialize
+      super
+      require 'fluent/plugin/socket_util'
+    end
+
+    config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
+    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :backlog, :integer, :default => nil
+    # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
+    config_param :linger_timeout, :integer, :default => 0
+    attr_reader :elapsed # for test
+
+    def configure(conf)
+      super
+
+      if element = conf.elements.first { |element| element.name == 'elapsed' }
+        tag = element["tag"] || 'elapsed'
+        interval = element["interval"].to_i || 60
+        hook = element['hook'] || 'on_message'
+        @elapsed = ElapsedMeasure.new(log, tag, interval, hook)
+      end
+    end
+
+    def start
+      @loop = Coolio::Loop.new
+
+      @lsock = listen
+      @loop.attach(@lsock)
+
+      @usock = SocketUtil.create_udp_socket(@bind)
+      @usock.bind(@bind, @port)
+      @usock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
+      @hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
+      @loop.attach(@hbr)
+
+      @thread = Thread.new(&method(:run))
+      @elapsed.start if @elapsed
+      @cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
+    end
+
+    def shutdown
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @usock.close
+      listen_address = (@bind == '0.0.0.0' ? '127.0.0.1' : @bind)
+      # This line is for connecting listen socket to stop the event loop.
+      # We should use more better approach, e.g. using pipe, fixing cool.io with timeout, etc.
+      TCPSocket.open(listen_address, @port) {|sock| } # FIXME @thread.join blocks without this line
+      @thread.join
+      @lsock.close
+      @elapsed.stop if @elapsed
+    end
+
+    class ElapsedMeasure
+      attr_reader :tag, :interval, :hook, :times, :sizes, :mutex, :thread, :log
+      def initialize(log, tag, interval, hook)
+        @log = log
+        @tag = tag
+        @interval = interval
+        @hook = hook.split(',')
+        @times = []
+        @sizes = []
+        @mutex = Mutex.new
+      end
+
+      def add(time, size)
+        @times << time
+        @sizes << size
+      end
+
+      def clear
+        @times.clear
+        @sizes.clear
+      end
+
+      def hookable?(caller)
+        @hook.include?(caller.to_s)
+      end
+
+      def measure_time(caller, size)
+        if hookable?(caller)
+          started = Time.now
+          yield
+          elapsed = (Time.now - started).to_f
+          log.debug "in_forward: elapsed time at #{caller} is #{elapsed} sec for #{size} bytes"
+          @mutex.synchronize { self.add(elapsed, size) }
+        else
+          yield
+        end
+      end
+
+      def start
+        @thread = Thread.new(&method(:run))
+      end
+
+      def stop
+        @thread.terminate
+        @thread.join
+      end
+
+      def run
+        @last_checked ||= Engine.now
+        while (sleep 0.5)
+          begin
+            now = Engine.now
+            if now - @last_checked >= @interval
+              flush(now)
+              @last_checked = now
+            end
+          rescue => e
+            log.warn "in_forward: #{e.class} #{e.message} #{e.backtrace.first}"
+          end
+        end
+      end
+
+      def flush(now)
+        times, sizes = [], []
+        @mutex.synchronize do
+          times = @times.dup
+          sizes = @sizes.dup
+          self.clear
+        end
+        if !times.empty? and !sizes.empty?
+          num = times.size
+          max = num == 0 ? 0 : times.max
+          avg = num == 0 ? 0 : times.map(&:to_f).inject(:+) / num.to_f
+          size_max = num == 0 ? 0 : sizes.max
+          size_avg = num == 0 ? 0 : sizes.map(&:to_f).inject(:+) / num.to_f
+          Engine.emit(@tag, now, {:num => num, :max => max, :avg => avg, :size_max => size_max, :size_avg => size_avg})
+        end
+      end
+    end
+
+    def listen
+      log.info "listening fluent socket on #{@bind}:#{@port}"
+      s = Coolio::TCPServer.new(@bind, @port, Handler, @linger_timeout, log, method(:on_message), @elapsed)
+      s.listen(@backlog) unless @backlog.nil?
+      s
+    end
+
+    #config_param :path, :string, :default => DEFAULT_SOCKET_PATH
+    #def listen
+    #  if File.exist?(@path)
+    #    File.unlink(@path)
+    #  end
+    #  FileUtils.mkdir_p File.dirname(@path)
+    #  log.debug "listening fluent socket on #{@path}"
+    #  Coolio::UNIXServer.new(@path, Handler, method(:on_message))
+    #end
+
+    def run
+      @loop.run
+    rescue => e
+      log.error "unexpected error", :error => e, :error_class => e.class
+      log.error_backtrace
+    end
+
+    protected
+    # message Entry {
+    #   1: long time
+    #   2: object record
+    # }
+    #
+    # message Forward {
+    #   1: string tag
+    #   2: list<Entry> entries
+    # }
+    #
+    # message PackedForward {
+    #   1: string tag
+    #   2: raw entries  # msgpack stream of Entry
+    # }
+    #
+    # message Message {
+    #   1: string tag
+    #   2: long? time
+    #   3: object record
+    # }
+    def on_message(msg)
+      if msg.nil?
+        # for future TCP heartbeat_request
+        return
+      end
+
+      # TODO format error
+      tag = msg[0].to_s
+      entries = msg[1]
+
+      if entries.class == String
+        # PackedForward
+        bytesize = tag.bytesize + entries.bytesize
+        measure_time(:on_message, bytesize) do
+          es = MessagePackEventStream.new(entries, @cached_unpacker)
+          Engine.emit_stream(tag, es)
+        end
+
+      elsif entries.class == Array
+        # Forward
+        es = MultiEventStream.new
+        entries.each {|e|
+          record = e[1]
+          next if record.nil?
+          time = e[0].to_i
+          time = (now ||= Engine.now) if time == 0
+          es.add(time, record)
+        }
+        measure_time(:on_message, 0) do
+          Engine.emit_stream(tag, es)
+        end
+
+      else
+        # Message
+        record = msg[2]
+        return if record.nil?
+        time = msg[1]
+        time = Engine.now if time == 0
+        bytesize = time.size + record.to_s.bytesize
+        measure_time(:on_message, bytesize) do
+          Engine.emit(tag, time, record)
+        end
+      end
+    end
+
+    def measure_time(caller, size)
+      @elapsed ? @elapsed.measure_time(caller, size) { yield } : yield
+    end
+
+    class Handler < Coolio::Socket
+      def initialize(io, linger_timeout, log, on_message, elapsed)
+        super(io)
+        if io.is_a?(TCPSocket)
+          opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
+          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+        end
+        @on_message = on_message
+        @log = log
+        @log.trace {
+          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
+          "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
+        }
+        @elapsed = elapsed
+      end
+
+      def on_connect
+      end
+
+      def on_read(data)
+        first = data[0]
+        if first == '{' || first == '['
+          m = method(:on_read_json)
+          @y = Yajl::Parser.new
+          @y.on_parse_complete = @on_message
+        else
+          m = method(:on_read_msgpack)
+          @u = MessagePack::Unpacker.new
+        end
+
+        (class << self; self; end).module_eval do
+          define_method(:on_read, m)
+        end
+        m.call(data)
+      end
+
+      def measure_time(caller, size)
+        @elapsed ? @elapsed.measure_time(caller, size) { yield } : yield
+      end
+
+      def on_read_json(data)
+        measure_time(:on_read, data.bytesize) do
+          @y << data
+        end
+      rescue => e
+        @log.error "forward error", :error => e, :error_class => e.class
+        @log.error_backtrace
+        close
+      end
+
+      def on_read_msgpack(data)
+        measure_time(:on_read, data.bytesize) do
+          @u.feed_each(data, &@on_message)
+        end
+      rescue => e
+        @log.error "forward error", :error => e, :error_class => e.class
+        @log.error_backtrace
+        close
+      end
+
+      def on_close
+        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
+      end
+    end
+
+    class HeartbeatRequestHandler < Coolio::IO
+      def initialize(io, callback)
+        super(io)
+        @io = io
+        @callback = callback
+      end
+
+      def on_readable
+        begin
+          msg, addr = @io.recvfrom(1024)
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
+          return
+        end
+        host = addr[3]
+        port = addr[1]
+        @callback.call(host, port, msg)
+      rescue
+        # TODO log?
+      end
+    end
+
+    def on_heartbeat_request(host, port, msg)
+      #log.trace "heartbeat request from #{host}:#{port}"
+      begin
+        @usock.send "\0", 0, host, port
+      rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR
+      end
+    end
+  end
+end

data/fluent-plugin-measure_time.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-measure_time"
+  gem.version     = "0.1.0"
+  gem.authors     = ["Naotoshi Seo"]
+  gem.email       = "sonots@gmail.com"
+  gem.homepage    = "https://github.com/sonots/fluent-plugin-measure_time"
+  gem.description = "Fluentd plugin to measure elapsed time to process messages"
+  gem.summary     = gem.description
+  gem.licenses    = ["MIT"]
+  gem.has_rdoc    = false
+
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry-nav"
+end

data/lib/fluent/plugin/in_measure_time.rb

@@ -0,0 +1,135 @@
+require 'fluent/input'
+
+module Fluent
+  class MeasureTimeInput < Input
+    Plugin.register_input('measure_time', self)
+
+    def configure(conf)
+      ::Fluent::Input.__send__(:include, MeasureTimable)
+      ::Fluent::Output.__send__(:include, MeasureTimable)
+    end
+  end
+
+  module MeasureTimable
+    def self.included(klass)
+      unless klass.method_defined?(:configure_without_measure_time)
+        klass.__send__(:alias_method, :configure_without_measure_time, :configure)
+        klass.__send__(:alias_method, :configure, :configure_with_measure_time)
+      end
+    end
+
+    attr_reader :measure_time
+
+    def configure_with_measure_time(conf)
+      configure_without_measure_time(conf)
+      if element = conf.elements.select { |element| element.name == 'measure_time' }.first
+        @measure_time = MeasureTime.new(self, log)
+        @measure_time.configure(element)
+      end
+    end
+  end
+
+  class MeasureTime
+    attr_reader :plugin, :log, :times, :mutex, :thread, :tag, :interval, :hook
+    def initialize(plugin, log)
+      @plugin = plugin
+      @klass = @plugin.class
+      @log = log
+      @times = []
+      @mutex = Mutex.new
+    end
+
+    def configure(conf)
+      @tag = conf['tag'] || 'measure_time'
+      unless @hook = conf['hook']
+        raise Fluent::ConfigError, '`hook` option must be specified in <measure_time></measure_time> directive'
+      end
+      @hook_msg = {"class" => @klass, "hook" => @hook, "object_id" => @plugin.object_id}
+      @interval = conf['interval'].to_i if conf['interval']
+      @add_or_emit_proc =
+        if @interval
+          # add to calculate statistics in each interval
+          Proc.new {|elapsed|
+            @mutex.synchronize { @times << elapsed }
+          }
+        else
+          # emit information immediately
+          Proc.new {|elapsed|
+            msg = {"time" => elapsed}.merge(@hook_msg)
+            ::Fluent::Engine.emit(@tag, ::Fluent::Engine.now, msg)
+          }
+        end
+      apply_hook
+    end
+
+    def apply_hook
+      @plugin.instance_eval <<EOF
+        def #{@hook}(*args)
+          measure_time.measure_time do
+            super
+          end
+        end
+        def start
+          super
+          measure_time.start
+        end
+        def stop
+          super
+          measure_time.stop
+        end
+EOF
+    end
+
+    def measure_time
+      started = Time.now
+      output = yield
+      elapsed = (Time.now - started).to_f
+      log.debug "elapsed time at #{@klass}##{@hook} is #{elapsed} sec"
+      @add_or_emit_proc.call(elapsed)
+      output
+    end
+
+    def start
+      return unless @interval
+      @thread = Thread.new(&method(:run))
+    end
+
+    def stop
+      return unless @interval
+      @thread.terminate
+      @thread.join
+    end
+
+    def run
+      @last_checked ||= Engine.now
+      while (sleep 0.5)
+        begin
+          now = Engine.now
+          if now - @last_checked >= @interval
+            flush(now)
+            @last_checked = now
+          end
+        rescue => e
+          log.warn "in_measure_time: hook #{@klass}##{@hook} #{e.class} #{e.message} #{e.backtrace.first}"
+        end
+      end
+    end
+
+    def flush(now)
+      times = []
+      @mutex.synchronize do
+        times = @times.dup
+        @times.clear
+      end
+      triple = nil
+      unless times.empty?
+        num = times.size
+        max = num == 0 ? 0 : times.max
+        avg = num == 0 ? 0 : times.map(&:to_f).inject(:+) / num.to_f
+        triple = [@tag, now, {:max => max, :avg => avg, :num => num}.merge(@hook_msg)]
+        Engine.emit(*triple)
+      end
+      triple
+    end
+  end
+end

data/spec/in_measure_time_spec.rb

@@ -0,0 +1,129 @@
+# encoding: UTF-8
+require_relative 'spec_helper'
+require 'fluent/plugin/in_forward'
+require 'fluent/plugin/out_stdout'
+
+describe Fluent::MeasureTimeInput do
+  before { Fluent::Test.setup }
+
+  def create_driver(conf=%[])
+    Fluent::Test::InputTestDriver.new(Fluent::MeasureTimeInput).configure(conf)
+  end
+
+  describe 'test configure' do
+    it { expect { create_driver }.not_to raise_error }
+  end
+end
+
+describe "extends Fluent::ForwardInput" do
+  before { Fluent::Test.setup }
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
+  end
+
+  def connect
+    TCPSocket.new('127.0.0.1', PORT)
+  end
+
+  def send_data(data)
+    io = connect
+    begin
+      io.write data
+    ensure
+      io.close
+    end
+  end
+
+  def self.unused_port
+    s = TCPServer.open(0)
+    port = s.addr[1]
+    s.close
+    port
+  end
+
+  PORT = unused_port
+  CONFIG = %[
+    port #{PORT}
+    bind 127.0.0.1
+  ]
+
+  let(:driver) { create_driver(config) }
+
+  describe 'test configure' do
+    let(:config) {CONFIG + %[
+      <measure>
+        tag test
+        interval 10
+        hook on_message
+      </measure>
+    ]}
+    let(:subject) { driver.instance.measure }
+    its(:tag) { should == 'test' }
+    its(:interval) { should == 10 }
+    its(:hook) { should == 'on_message' }
+  end
+
+  describe 'test emit' do
+    let(:config) {CONFIG + %[
+      <measure>
+        tag measure
+        interval 1
+        # hook Fluent::ForwardInput::Handler.on_read # not support inner class yet
+        hook Fluent::ForwardInput.on_message
+      </measure>
+    ]}
+    it 'should flush' do
+      d = driver.instance
+      d.__send__(:on_message, ['tag1', 0, {'a'=>1}].to_msgpack)
+      triple = d.measure.flush(0)
+      triple[0].should == 'measure'
+      triple[2].keys.should =~ [:num, :max, :avg]
+    end
+  end
+end
+
+describe "extends Fluent::StdoutOutput" do
+  before { Fluent::Test.setup }
+
+  def create_driver(conf=CONFIG, tag = 'test')
+    Fluent::Test::OutputTestDriver.new(Fluent::StdoutOutput, tag).configure(conf)
+  end
+
+  CONFIG = %[
+  ]
+
+  let(:driver) { create_driver(config) }
+
+  describe 'test configure' do
+    let(:config) {CONFIG + %[
+      <measure>
+        tag test
+        interval 10
+        hook emit
+      </measure>
+    ]}
+    let(:subject) { driver.instance.instance_variable_get(:@measure) }
+    its(:tag) { should == 'test' }
+    its(:interval) { should == 10 }
+    its(:hook) { should == 'emit' }
+  end
+
+  describe 'test emit' do
+    let(:config) {CONFIG + %[
+      <measure>
+        tag measure
+        interval 1
+        hook emit
+      </measure>
+    ]}
+    it 'should flush' do
+      d = driver.instance
+      d.emit('tag1', Fluent::OneEventStream.new(0, {'a'=>1}), Fluent::NullOutputChain.instance)
+      triple = d.measure.flush(0)
+      triple[0].should == 'measure'
+      triple[2].keys.should =~ [:num, :max, :avg]
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+require 'rubygems'
+require 'bundler'
+require 'fluent/load'
+Bundler.setup(:default, :test)
+Bundler.require(:default, :test)
+
+require 'fluent/test'
+require 'rspec'
+require 'pry'
+
+$TESTING=true
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'fluent/plugin/in_measure_time'

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-measure_time
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Naotoshi Seo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to measure elapsed time to process messages
+email: sonots@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- benchmark/Gemfile
+- benchmark/README.md
+- benchmark/agent.conf
+- benchmark/dummer.conf
+- benchmark/hooked_in_forward.conf
+- benchmark/in_forward.conf
+- benchmark/patched_in_forward.conf
+- benchmark/plugin/in_forward.rb
+- fluent-plugin-measure_time.gemspec
+- lib/fluent/plugin/in_measure_time.rb
+- spec/in_measure_time_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/sonots/fluent-plugin-measure_time
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to measure elapsed time to process messages
+test_files:
+- spec/in_measure_time_spec.rb
+- spec/spec_helper.rb