fluent-plugin-werkzeug-profiler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 739f2096bb9c486f62f7a66100ceedc3b73ef5ab
+  data.tar.gz: 3819d28c2bfdf620583bbdbecc5ec20c720fca84
+SHA512:
+  metadata.gz: 98abcf85d13e7656fad288229e04e5c619eedc9105da91995df78ce00fc1386a4f3c1b143417ee0ca1fac895eee7e6ab95d9864a2cfeaa9ad0308a5393ad3efa
+  data.tar.gz: dbf2f1a901ff19f750475c9b6873e915773ca8ead4d59a86cff99e8c6917d10bb4f9f2b5649681963a94c69d9cfe082cd1069e509f9ea20595268ed00996dc57

data/.gitignore

@@ -0,0 +1,46 @@
+# Created by http://gitignore.io
+
+### OSX ###
+.DS_Store
+.AppleDouble
+.LSOverride
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear on external disk
+.Spotlight-V100
+.Trashes
+
+### vim ###
+.*.s[a-w][a-z]
+*.un~
+Session.vim
+.netrwhist
+*~
+
+### Ruby ###
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-werkzeug-profiler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Kenta MORI
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# fluent-plugin-werkzeug-profiler
+
+## Component
+
+### WerkzeugProfilerInput
+
+Fluent input plugin for Werkzeug WSGI application profiler statistics.
+
+For example, Werkzeug profiler output following log.
+```
+----------------------------------------
+PATH: '/'
+         15724 function calls (14741 primitive calls) in 0.418 seconds
+
+   Ordered by: internal time, call count
+   List reduced from 490 to 30 due to restriction <30>
+
+   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+        2    0.226    0.113    0.226    0.113 {method 'query' of '_mysql.connection' objects}
+        1    0.000    0.000    0.026    0.026 build/bdist.linux-x86_64/egg/MySQLdb/__init__.py:78(Connect)
+    99/31    0.000    0.000    0.000    0.000 /usr/lib/python2.7/sre_parse.py:141(getwidth)
+
+----------------------------------------
+```
+Results are as follows.
+```
+{ "uri"=>"/", "tot_ncalls"=>"2", "prim_ncalls"=>"2", "tottime"=>"0.226", 
+  "tot_percall"=>"0.113", "cumtime"=>"0.226", "cum_percall"=>"0.113", 
+  "filename:lineno(function)"=>"{method 'query' of '_mysql.connection' objects}" }, 
+{ "uri"=>"/", "tot_ncalls"=>"1", "prim_ncalls"=>"1", "tottime"=>"0.000", 
+  "tot_percall"=>"0.000", "cumtime"=>"0.026", "cum_percall"=>"0.026", 
+  "filename:lineno(function)"=>"build/bdist.linux- x86_64/egg/MySQLdb/__init__.py:78(Connect)" }, 
+{ "uri"=>"/", "tot_ncalls"=>"99", "prim_ncalls"=>"31", "tottime"=>"0.000", 
+"tot_percall"=>"0.000", "cumtime"=>"0.000", "cum_percall"=>"0.000", 
+    ["filename:lineno(function)"=>"/usr/lib/python2.7/sre_parse.py:141(getwidth)" }
+```
+
+## Installation
+
+```
+$ gem install fluent-plugin-werkzeug-profiler
+```
+
+## Configuration
+
+### WerkzeugProfilerInput
+
+```
+<source>
+  type werkzeug
+  path path/to/werkzeug.log
+  tag werkzeug.webserver
+  time_format %d-%b-%Y:%H:%M:%S
+</source>
+```
+
+## License
+
+This software is released under the MIT License, see LICENSE.txt.

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+
+require 'fileutils'
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-werkzeug-profiler.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-werkzeug-profiler"
+  gem.version       = "0.0.1"
+  gem.authors       = ["Kenta MORI"]
+  gem.email         = ["zoncoen@gmail.com"]
+  gem.description   = %q{Fluent input plugin for Werkzeug WSGI application profiler statistics.}
+  gem.summary       = %q{Fluent input plugin for Werkzeug WSGI application profiler statistics.}
+  gem.homepage      = "https://github.com/zoncoen/fluent-plugin-werkzeug-profiler.git"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "rake"
+  gem.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/in_werkzeug_profiler.rb

@@ -0,0 +1,94 @@
+module Fluent
+
+  class WerkzeugProfilerInput < TailInput
+    Plugin.register_input('werkzeug', self)
+
+    # get time format from config
+    config_param :time_format, :string, :default => '%d/%b/%Y:%H:%M:%S %z'
+
+    # override
+    def configure_parser(conf)
+      @parser = WerkzeugProfilerParser.new(@time_format)
+    end
+
+    # override
+    def receive_lines(lines)
+      es = MultiEventStream.new
+
+      for line in @parser.divide(lines) do
+        begin
+          time, record = parse_line(line)
+          if time && record
+            es.add(time, record)
+          end
+        rescue
+          $log.warn line.dump, :error=>$!.to_s
+          $log.debug_backtrace
+        end
+      end
+
+      unless es.empty?
+        Engine.emit_stream(@tag, es)
+      end
+    end
+
+    class WerkzeugProfilerParser
+      def initialize(time_format)
+        @time_format = time_format
+        @keys = ['uri', 'tot_ncalls', 'prim_ncalls', 'tottime', 'tot_percall', 'cumtime', 'cum_percall', 'filename:lineno(function)']
+        @record_size = @keys.length - 2
+        @path = 'path'
+        @time = 'time'
+      end
+
+      def parse(values)
+        time = values.shift
+        array = [@keys,values].transpose.flatten
+        record = Hash[*array]
+        return time, record
+      end
+
+      def divide(lines)
+        records = []
+        # remove empty lines
+        lines = lines.join('').gsub(/\n\n/, "\n").split("\n")
+
+        for line in lines do
+          if line.start_with?('-')
+            @path = nil
+            next
+          end
+
+          if line.start_with?('PATH:')
+            path = line.split(nil)[1]
+            @path = path[1..path.length-2]
+            @time = Time.now.strftime(@time_format)
+            next
+          end
+
+          if line.strip.split(nil)[1] == 'function' || line.strip.start_with?('ncalls', 'Ordered', 'List')
+            next
+          elsif @path != nil
+            values = line.split(nil)
+            # divide ncalls
+            ncalls = values[0].split('/')
+            values[0] = ncalls[0]
+            values[1,0] = ncalls[ncalls.length-1]
+            if values.length > @record_size + 1
+              values[@record_size] = values[@record_size..values.length-1].join(' ')
+            end
+            record = values[0..@record_size]
+            record.unshift(@path)
+            record.unshift(@time)
+            records << record
+            next
+          end
+        end
+
+        # return records as array
+        records
+      end
+    end
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/in_tail'
+require 'fluent/plugin/in_werkzeug_profiler'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_werkzeug_profiler.rb

@@ -0,0 +1,68 @@
+require 'helper'
+require 'fileutils'
+
+class WerkzeugProfilerInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  def teardown
+    FileUtils.rm_rf(TMP_DIR)
+  end
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp"
+
+  CONFIG = %[
+    path #{TMP_DIR}/werkzeug-profiler-test.txt
+    time_format %d-%b-%Y:%H:%M:%S
+    tag test
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::WerkzeugProfilerInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal ["#{TMP_DIR}/werkzeug-profiler-test.txt"], d.instance.paths
+    assert_equal "%d-%b-%Y:%H:%M:%S", d.instance.time_format
+  end
+
+  def test_emit
+    File.open("#{TMP_DIR}/werkzeug-profiler-test.txt", "w") {|f|
+      f.puts "old logs."
+    }
+
+    d = create_driver
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/werkzeug-profiler-test.txt", "a") {|f|
+        f.puts "--------------------------------------------------------------------------------"
+        f.puts "PATH: '/'"
+        f.puts "         15724 function calls (14741 primitive calls) in 0.418 seconds"
+        f.puts ""
+        f.puts "   Ordered by: internal time, call count"
+        f.puts "   List reduced from 490 to 30 due to restriction <30>"
+        f.puts ""
+        f.puts "   ncalls  tottime  percall  cumtime  percall filename:lineno(function)"
+        f.puts "        2    0.226    0.113    0.226    0.113 {method 'query' of '_mysql.connection' objects}"
+        f.puts "        1    0.000    0.000    0.026    0.026 build/bdist.linux-x86_64/egg/MySQLdb/__init__.py:78(Connect)"
+        f.puts "    99/31    0.000    0.000    0.000    0.000 /usr/lib/python2.7/sre_parse.py:141(getwidth)"
+        f.puts ""
+        f.puts "--------------------------------------------------------------------------------"
+        f.puts ""
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"uri"=>"/", "tot_ncalls"=>"2", "prim_ncalls"=>"2", "tottime"=>"0.226", "tot_percall"=>"0.113", "cumtime"=>"0.226", "cum_percall"=>"0.113", "filename:lineno(function)"=>"{method 'query' of '_mysql.connection' objects}"}, emits[0][2])
+    assert_equal({"uri"=>"/", "tot_ncalls"=>"1", "prim_ncalls"=>"1", "tottime"=>"0.000", "tot_percall"=>"0.000", "cumtime"=>"0.026", "cum_percall"=>"0.026", "filename:lineno(function)"=>"build/bdist.linux-x86_64/egg/MySQLdb/__init__.py:78(Connect)"}, emits[1][2])
+    assert_equal({"uri"=>"/", "tot_ncalls"=>"99", "prim_ncalls"=>"31", "tottime"=>"0.000", "tot_percall"=>"0.000", "cumtime"=>"0.000", "cum_percall"=>"0.000", "filename:lineno(function)"=>"/usr/lib/python2.7/sre_parse.py:141(getwidth)"}, emits[2][2])
+  end
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-werkzeug-profiler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Kenta MORI
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluent input plugin for Werkzeug WSGI application profiler statistics.
+email:
+- zoncoen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-werkzeug-profiler.gemspec
+- lib/fluent/plugin/in_werkzeug_profiler.rb
+- test/helper.rb
+- test/plugin/test_in_werkzeug_profiler.rb
+homepage: https://github.com/zoncoen/fluent-plugin-werkzeug-profiler.git
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.0
+signing_key: 
+specification_version: 4
+summary: Fluent input plugin for Werkzeug WSGI application profiler statistics.
+test_files:
+- test/helper.rb
+- test/plugin/test_in_werkzeug_profiler.rb