fluent-plugin-watch-process 0.0.1 → 0.0.2

data/README.md

@@ -3,17 +3,17 @@ fluent-plugin-watch-process [![Build Status](https://travis-ci.org/y-ken/fluent-
 
 ## Overview
 
-Fluentd Input plugin to collect process information via ps command.
+Fluentd Input plugin to collect continual process information via ps command. It is useful for cron/barch process monitoring.
 
 ## Use Cases
 
-* collect cron/batch process for analysis.
+* collect cron/batch process for long term analysis.
   * high cpu load time
   * high usage of memory
   * determine too long running task
 
 * output destination example
-  * Elasticsearch + Kibana to visualize statistics. Example: [example1.conf](https://github.com/y-ken/fluent-plugin-watch-process/blob/master/example1.conf)
+  * Elasticsearch + Kibana to visualize cron/batch process statistics. Example: [example1.conf](https://github.com/y-ken/fluent-plugin-watch-process/blob/master/example1.conf)
   * save process information as audit log into AWS S3 which filename isolated by hostname. Example: [example2.conf](https://github.com/y-ken/fluent-plugin-watch-process/blob/master/example2.conf)
 
 ## Installation
@@ -51,37 +51,38 @@ After restarting td-agent, it will output process information to the td-agent.lo
 
 `````
 $ tail -f /var/log/td-agent/td-agent.log
+...snip...
 2014-01-16 14:21:34 +0900 debug.batch.localhost: {"start_time":"2014-01-16 14:21:13 +0900","user":"td-agent","pid":17486,"parent_pid":17483,"cpu_time":"00:00:00","cpu_percent":1.5,"memory_percent":3.5,"mem_rss":36068,"mem_size":60708,"state":"S","proc_name":"ruby","command":"/usr/lib64/fluent/ruby/bin/ruby /usr/sbin/td-agent --group td-agent --log /var/log/td-agent/td-agent.log --daemon /var/run/td-agent/td-agent.pid","elapsed_time":21}
 `````
 
 ### Syntax
 
-* tag # Required
+* tag (Required)
   * record output destination
   * supported tag placeholders are `${hostname}` and `__HOSTNAME__`.
 
-* command # Optional
+* command (Optional)
   * execute ps command with some options
   * [default] Linux: `LANG=en_US.UTF-8 && ps -ewwo lstart,user:20,pid,ppid,time,%cpu,%mem,rss,sz,s,comm,cmd`
   * [default] MacOSX: `LANG=en_US.UTF-8 && ps -ewwo lstart,user,pid,ppid,time,%cpu,%mem,rss,vsz,state,comm,command`
 
-* keys # Optional
+* keys (Optional)
   * output record keys of the ps command results
   * [default] start_time user pid parent_pid cpu_time cpu_percent memory_percent mem_rss mem_size state proc_name command
 
-* types # Optional
+* types (Optional)
   * settings of converting types from string to integer/float.
   * [default] pid:integer parent_pid:integer cpu_percent:float memory_percent:float mem_rss:integer mem_size:integer
 
-* interval # Optional
+* interval (Optional)
   * execute interval time
   * [default] 5s
 
-* lookup_user # Optional
+* lookup_user (Optional)
   * filter process owner username with comma delimited
   * [default] N/A
 
-* hostname_command # Optional
+* hostname_command (Optional)
   * settings for tag placeholder, `${hostname}` and `__HOSTNAME__`. By default, it using long hostname.
   * to use short hostname, set `hostname -s` for this option on linux/mac.
   * [default] `hostname`

data/fluent-plugin-watch-process.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-watch-process"
-  s.version     = "0.0.1"
+  s.version     = "0.0.2"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-watch-process"
-  s.summary     = %q{Fluentd Input plugin to collect process information via ps command.}
+  s.summary     = %q{Fluentd Input plugin to collect continual process information via ps command. It is useful for cron/barch process monitoring.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/in_watch_process.rb

@@ -10,6 +10,9 @@ module Fluent
     config_param :lookup_user, :string, :default => nil
     config_param :hostname_command, :string, :default => 'hostname'
 
+    DEFAULT_KEYS = %w(start_time user pid parent_pid cpu_time cpu_percent memory_percent mem_rss mem_size state proc_name command)
+    DEFAULT_TYPES = %w(pid:integer parent_pid:integer cpu_percent:float memory_percent:float mem_rss:integer mem_size:integer)
+
     Converters = {
       'string' => lambda { |v| v.to_s },
       'integer' => lambda { |v| v.to_i },
@@ -39,8 +42,8 @@ module Fluent
       super
 
       @command = @command || get_ps_command
-      @keys = @keys || %w(start_time user pid parent_pid cpu_time cpu_percent memory_percent mem_rss mem_size state proc_name command)
-      types = @types || %w(pid:integer parent_pid:integer cpu_percent:float memory_percent:float mem_rss:integer mem_size:integer)
+      @keys = @keys.nil? ? DEFAULT_KEYS : @keys.to_s.gsub(' ', '').split(',')
+      @types = @types || DEFAULT_TYPES
       @types_map = Hash[types.map{|v| v.split(':')}]
       @lookup_user = @lookup_user.gsub(' ', '').split(',') unless @lookup_user.nil?
       @interval = Config.time_value(@interval)
@@ -61,15 +64,20 @@ module Fluent
         io = IO.popen(@command, 'r')
         io.gets
         while result = io.gets
-          values = result.chomp.strip.split(/\s+/, @keys.size + 4)
-          time = Time.parse(values[0...5].join(' '))
+          keys_size = @keys.size
+          if result =~ /(?<lstart>(^\w+ \w+ \d+ \d\d:\d\d:\d\d \d+))/
+            lstart = Time.parse($~[:lstart])
+            values = result.sub($~[:lstart], '')
+            keys_size -= 1
+          end
+          values = values.chomp.strip.split(/\s+/, keys_size)
           data = Hash[
-            @keys.zip([time.to_s, values.values_at(5..15)].flatten).map do |k,v|
+            @keys.zip([lstart.to_s, values].reject(&:empty?).flatten).map do |k,v|
               v = Converters[@types_map[k]].call(v) if @types_map.include?(k)
               [k,v]
             end
           ]
-          data['elapsed_time'] = (Time.now - Time.parse(data['start_time'])).to_i
+          data['elapsed_time'] = (Time.now - Time.parse(data['start_time'])).to_i if data['start_time']
           next unless @lookup_user.nil? || @lookup_user.include?(data['user'])
           tag = @tag.gsub(/(\${[a-z]+}|__[A-Z]+__)/, get_placeholder)
           Engine.emit(tag, Engine.now, data)
@@ -77,6 +85,8 @@ module Fluent
         io.close
         sleep @interval
       end
+      rescue StandardError => e
+        $log.error "watch_process: error has occured. #{e.message}"
     end
 
     def get_ps_command

data/test/plugin/test_in_watch_process.rb

@@ -7,11 +7,10 @@ class WatchProcessInputTest < Test::Unit::TestCase
 
   CONFIG = %[
     tag          input.watch_process
-    lookup_user  apache, mycron
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::WatchProcessInput, tag).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::WatchProcessInput).configure(conf)
   end
 
   def test_configure
@@ -27,4 +26,3 @@ class WatchProcessInputTest < Test::Unit::TestCase
     assert_equal ['apache', 'mycron'], d.instance.lookup_user
   end
 end
-

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-watch-process
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-16 00:00:00.000000000 Z
+date: 2014-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -85,7 +85,8 @@ rubyforge_project:
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Fluentd Input plugin to collect process information via ps command.
+summary: Fluentd Input plugin to collect continual process information via ps command.
+  It is useful for cron/barch process monitoring.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_watch_process.rb