logstash-input-proc 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b39878e4ae4d55b561d66b98e5df734d9de7482
-  data.tar.gz: 7158cef5bf169956201c15bbd540aaeff5f1fb89
+  metadata.gz: 1a7521ce81c7e07305cdbef8c821907fa0ea7843
+  data.tar.gz: 64a7d10f0fae20db1bc98f6d7454451ea1083999
 SHA512:
-  metadata.gz: 53acb7d9a31f969d1aae2cc30bcf9766bbae43751a044f02f41f88e03af3b712482bac18011341bba4d794ffa79407aa8802c7f986448af42c1adad78d8c8855
-  data.tar.gz: e832d5ab85d648286d9ee9b58e66de06abf50843e3e57c8a59c901ab497b900956b1415bba3c70cbb05eebbdf8a44c7a86b9c00457002750d8bd59f583f3f685
+  metadata.gz: 80939c69de0a1520c46fdbe9606c6468714472879b0d30f70bdf7db816e7dae0a50343ac8088b2e54152e9454f1f52a573705d423c01cc0c387857da2d711540
+  data.tar.gz: 2d0651bd5ed03e5ce16b129e47cd5fc607547409e1747e5eb98d1fec6bdb6e4a90266fbcac02881d4ed4a6996a667c1c40312751d05571e527f41987c4901cf9

data/ElasticSearch/templates/pidstats.json

@@ -0,0 +1,185 @@
+{
+    "template": "pidstats*",
+    "order": 10,
+    "aliases": {
+        "pidstats": [
+            {
+                "add": {
+                    "index": "pidstats"
+                }
+            }
+        ]
+    },
+    "mappings": {
+        "pidstats": {
+                    "properties": {
+                        "uid": {
+                            "type": "long"
+                        },
+                        "tty_nr": {
+                            "type": "long"
+                        },
+                        "nlwp": {
+                            "type": "long"
+                        },
+                        "utime": {
+                            "type": "long"
+                        },
+                        "startcode": {
+                            "type": "long"
+                        },
+                        "rt_priority": {
+                            "type": "long"
+                        },
+                        "egid": {
+                            "type": "long"
+                        },
+                        "stime": {
+                            "type": "long"
+                        },
+                        "state": {
+                            "type": "string"
+                        },
+                        "sigcatch": {
+                            "type": "long"
+                        },
+                        "kstkeip": {
+                            "type": "long"
+                        },
+                        "nice": {
+                            "type": "long"
+                        },
+                        "signal": {
+                            "type": "long"
+                        },
+                        "nswap": {
+                            "type": "long"
+                        },
+                        "sigignore": {
+                            "type": "long"
+                        },
+                        "vsize": {
+                            "type": "long"
+                        },
+                        "pgrp": {
+                            "type": "long"
+                        },
+                        "endcode": {
+                            "type": "long"
+                        },
+                        "kstkesp": {
+                            "type": "long"
+                        },
+                        "priority": {
+                            "type": "long"
+                        },
+                        "name": {
+                            "type": "string"
+                        },
+                        "rss": {
+                            "type": "long"
+                        },
+                        "cmajflt": {
+                            "type": "long"
+                        },
+                        "blocked": {
+                            "type": "long"
+                        },
+                        "processor": {
+                            "type": "long"
+                        },
+                        "rlim": {
+                            "type": "string"
+                        },
+                        "tpgid": {
+                            "type": "long"
+                        },
+                        "euid": {
+                            "type": "long"
+                        },
+                        "comm": {
+                            "type": "string"
+                        },
+                        "starttime": {
+                            "type": "long"
+                        },
+                        "flags": {
+                            "type": "long"
+                        },
+                        "ppid": {
+                            "type": "long"
+                        },
+                        "cmdline": {
+                            "type": "string"
+                        },
+                        "environ": {
+                            "type": "object"
+                        },
+                        "pid": {
+                            "type": "long"
+                        },
+                        "fd": {
+                            "type": "object"
+                        },
+                        "startstack": {
+                            "type": "long"
+                        },
+                        "minflt": {
+                            "type": "long"
+                        },
+                        "io": {
+                            "type": "object"
+                        },
+                        "exit_signal": {
+                            "type": "long"
+                        },
+                        "cstime": {
+                            "type": "long"
+                        },
+                        "itrealvalue": {
+                            "type": "long"
+                        },
+                        "wchan": {
+                            "type": "string"
+                        },
+                        "session": {
+                            "type": "long"
+                        },
+                        "majflt": {
+                            "type": "long"
+                        },
+                        "gid": {
+                            "type": "long"
+                        },
+                        "policy": {
+                            "type": "long"
+                        },
+                        "cutime": {
+                            "type": "long"
+                        },
+                        "cnswap": {
+                            "type": "long"
+                        },
+                        "cminflt": {
+                            "type": "long"
+                        }
+                    }
+                },
+                "@timestamp": {
+                    "format": "dateOptionalTime",
+                    "type": "date"
+                },
+                "host": {
+                    "type": "string"
+                },
+                "file": {
+                    "type": "string"
+                },
+                "type": {
+                    "type": "string"
+                },
+                "@version": {
+                    "type": "string"
+                }
+            }
+}

data/README.md

@@ -1,130 +1,20 @@
 ![](https://github.com/eperry/logstash-input-proc/wiki/MemInfoDashboard.png)
 
-# Logstash Plugin
+# Logstash Input Proc Parser
+##[Home page](http://eperry.github.io/logstash-input-proc/)
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for with [Logstash](https://github.com/elasticsearch/logstash).
 
 This plugin is to read the /proc virtual file system , decode the files in it.
 I am using the following pages for reference 
 
-- http://man7.org/linux/man-pages/man5/proc.5.html
-
-
-
-
 ## Documentation
+[Documentation](https://github.com/eperry/logstash-input-proc/wiki/documentation)
 
+[Quickstart](https://github.com/eperry/logstash-input-proc/wiki/quickstart)
 
-### 1. Plugin Developement and Testing
-
-#### Code
-- To get started, you'll need JRuby with the Bundler gem installed.
- ```sh
-bundle install
-```
-
-- Then clone this repo
-- You will need to either clone the logstash repo or download the binary
-
-
-
-### 2. Running the unpublished Plugin in Logstash
-
-#### 2.1 Run in a local Logstash clone
-
-- Edit Logstash `Gemfile` and add the local plugin path, for example:
-```ruby
-gem "logstash-input-proc", :path => "/your/local/logstash-input-proc"
-```
-- Install plugin
-```sh
-bin/plugin install --no-verify
-```
-- install Ruby Debug
-```sh
-bin/plugin install logstash-codec-rubydebug
-```
-- Run Logstash with your plugin
-```sh
-bin/logstash -e 'input {proc {interval=>60}}  output { stdout{ codec=>"rubydebug"}}'
-```
-At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
-
-#### 2.2 Run in an installed Logstash
-
-You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
-
-- Build your plugin gem
-```sh
-gem build logstash-output-proc.gemspec
-```
-- Install the plugin from the Logstash home
-```sh
-bin/plugin install /your/local/plugin/logstash-input-proc.gem
-```
-- Start Logstash and proceed to test the plugin
-- 
-# Example Config all features enabled
-```ruby
-input {
-    proc {
-        interval=>60
-        vmstats =>{ }
-        loadavg =>{ }
-        meminfo =>{ }
-        pidstats =>{ 
-            user => "root"
-        }
-        
-    }
-}
-
-output { 
-    stdout{ 
-        codec=>"rubydebug"
-    }
-}
-```
-#Example Minimal
-
-```ruby
-input {
-    proc {
-        interval=>60
-        meminfo =>{ }
-    }
-}
-
-output { 
-    stdout{ 
-        codec=>"rubydebug"
-    }
-}
-```
-
-## 3.0 Kibana Dashboards
-
-Still a work in progress but I have saved a copy of the Kibana 4.1 dashboards I have created
-in the ~/Kibana Directory, you should be able to import them from the Kibana->settings->Objects pages
-
-These dashboards are right now a way of me validating the data loaded in elasticsearch is usable and provide an example for others to work off of. They work with the setup
-of elasticsearch as defined below.
-
-
-## 4.0 Elasticsearch Templates
-
-In the ~/ElasitcSearch Directory are all the Elasticsearch templates I am developing to work with this plugin. 
-While they may not be exactly what you need they are a good start. 
-
-I load them via the ${ES_HOME/config/templates directory but feel free to load them in your preffered way 
+[Development](https://github.com/eperry/logstash-input-proc/wiki/development)
 
-These templates are based on the fact that your indexes for the data are created like so:
-```
-output { 
-    elasticsearch { 
-       host => localhost 
-       index => "%{type}-%{+YYYY.MM.dd}"
-    }
-}
-```
+[kibana](https://github.com/eperry/logstash-input-proc/wiki/kibana)
 
+[ElasticSearch](https://github.com/eperry/logstash-input-proc/wiki/es)

data/documentation.md

@@ -0,0 +1,6 @@
+This plugin is to read the /proc virtual file system , decode the files in it.
+I am using the following pages for reference 
+
+- http://man7.org/linux/man-pages/man5/proc.5.html
+
+

data/lib/logstash/inputs/proc.rb

@@ -13,7 +13,7 @@ class LogStash::Inputs::Proc < LogStash::Inputs::Base
   config_name "proc"
 
   # If undefined, Logstash will complain, even if codec is unused.
-  default :codec, "plain" 
+  default :codec, "json" 
 
   # The message string to use in the event.
   #config :message, :validate => :string, :default => "Hello World!"
@@ -106,7 +106,6 @@ def readPidStats(queue)
     fuid = Etc.getpwnam(@pidstats["user"]).uid
     @logger.info? && @logger.info("Filtering userid =" + @pidstats["user"] )
   end
-  process = Hash.new
   #Loosely based on the GEM ProcTable  which was  based on the Perl Module ProcTable
      Dir.foreach("/proc"){ |file|
         next if file =~ /\D/ # Skip non-numeric directories
@@ -114,24 +113,24 @@ def readPidStats(queue)
           fileUid = File.stat("/proc/"+file).uid
           next if fileUid != fuid
         end
-        
+        process = Hash.new
         # Get /proc/<pid>/cmdline information. Strip out embedded nulls.
         begin
           data = IO.read("/proc/#{file}/cmdline").tr("\000", ' ').strip
           process["cmdline"] = data 
         rescue
-          next # Process terminated, on to the next process
+          # Ignore and move on.
         end
-
         # Get /proc/<pid>/cwd information
-        process["cwd"] = File.readlink("/proc/#{file}/cwd") rescue nil
+        process["cwd"] = File.readlink("/proc/#{file}/cwd") rescue 
 
         # Get /proc/<pid>/environ information. Environment information
         # is represented as a Hash, with the environment variable as the
         # key and its value as the hash value.
-        process["environ"] = Hash.new
+
 
         begin
+          process["environ"] = Hash.new
           IO.read("/proc/#{file}/environ").split("\0").each{ |str|
             key, value = str.split('=')
             process["environ"][key] = value
@@ -141,7 +140,7 @@ def readPidStats(queue)
         end
 
         # Get /proc/<pid>/exe information
-        process["exe"] = File.readlink("/proc/#{file}/exe") rescue nil
+        process["exe"] = File.readlink("/proc/#{file}/exe") rescue 
 
         # Get /proc/<pid>/fd information. File descriptor information
         # is represented as a Hash, with the fd as the key, and its
@@ -150,23 +149,41 @@ def readPidStats(queue)
 
         begin
           Dir.foreach("/proc/#{file}/fd/") { |fd|
-            process["fd"][fd] = File.readlink("/proc/#{file}/fd/"+fd)  rescue nil
+            process["fd"][fd] = File.readlink("/proc/#{file}/fd/"+fd)  rescue process["fd"] = []
           }
-          rescue 
-          process["fd"] = ""
+        rescue 
           #  # Ignore and move on
         end
 
         # Get /proc/<pid>/root information
-        process["root"] = File.readlink("/proc/#{file}/root") rescue nil
-
-        # Get /proc/<pid>/stat information
-        stat = IO.read("/proc/#{file}/stat") rescue next
+        process["root"] = File.readlink("/proc/#{file}/root") rescue 
 
         # Get number of LWP, one directory for each in /proc/<pid>/task/
         # Every process has at least one thread, so if we fail to read the task directory, set nlwp to 1.
-        process["nlwp"] = Dir.glob("/proc/#{file}/task/*").length rescue process["nlwp"] = 1
+        process["nlwp"] = Dir.glob("/proc/#{file}/task/*").length rescue process["nlwp"] = 1 
+        
+        # cat /proc/3828/io
+        #          rchar: 323934931
+        #          wchar: 323929600
+        #          syscr: 632687
+        #          syscw: 632675
+        #          read_bytes: 0
+        #          write_bytes: 323932160
+        #          cancelled_write_bytes: 0
+        begin
+          process["io"] = Hash.new
 
+          IO.foreach("/proc/#{file}/io") do |line|
+              key, value = line.split(/[:\s]+/)
+              process["io"][key] = value.to_i
+          end
+          rescue 
+          # Ignore and move on.
+        end
+
+        # Get /proc/<pid>/stat information
+      begin
+        stat = IO.read("/proc/#{file}/stat") 
         # Deal with spaces in comm name. Courtesy of Ara Howard.
         re = %r/\([^\)]+\)/
         comm = stat[re]
@@ -174,7 +191,6 @@ def readPidStats(queue)
         stat[re] = comm
 
         stat = stat.split
-
         process["pid"]         = stat[0].to_i
         process["comm"]        = stat[1].tr('()','') # Remove parens
         process["state"]       = stat[2]
@@ -199,7 +215,7 @@ def readPidStats(queue)
         process["starttime"]   = stat[21].to_i
         process["vsize"]       = stat[22].to_i
         process["rss"]         = stat[23].to_i
-        process["rlim"]        = stat[24].to_i
+        process["rlim"]        = stat[24].to_s
         process["startcode"]   = stat[25].to_i
         process["endcode"]     = stat[26].to_i
         process["startstack"]  = stat[27].to_i
@@ -209,13 +225,16 @@ def readPidStats(queue)
         process["blocked"]     = stat[31].to_i
         process["sigignore"]   = stat[32].to_i
         process["sigcatch"]    = stat[33].to_i
-        process["wchan"]       = stat[34].to_i
+        process["wchan"]       = stat[34].to_s
         process["nswap"]       = stat[35].to_i
         process["cnswap"]      = stat[36].to_i
         process["exit_signal"] = stat[37].to_i
         process["processor"]   = stat[38].to_i
         process["rt_priority"] = stat[39].to_i
         process["policy"]      = stat[40].to_i
+      rescue
+        
+      end
         # Get /proc/<pid>/status information (name, uid, euid, gid, egid)
         begin
           IO.foreach("/proc/#{file}/status") do |line|
@@ -231,17 +250,13 @@ def readPidStats(queue)
             end
           end
         rescue Errno::ESRCH, Errno::ENOENT
-          next
         end
         
         # If cmdline is empty use comm instead
         process["cmdline"] = process["comm"] if process["cmdline.empty?"]
-
-        
         event = LogStash::Event.new( "file" => "/proc" ,"host" => @host, "type" => "pidstats" , "process" => process);
         decorate(event)
         queue << event
-
       }
 
 
@@ -462,7 +477,6 @@ def readWireless(queue)
     lines.each { |line|
       #@logger.info? && @logger.info("LINE: "+line)
       m = line.strip.split(/[:\s]+/)
-      #puts(m)
       if (m && m.length >= 11 )
       event = LogStash::Event.new(
               "raw"           => line, 
@@ -553,7 +567,7 @@ end
         sleep(sleeptime)
       end
       rescue => exception
-        #puts exception.message 
+        puts exception.message 
         puts exception.backtrace
         raise
       end # rescue

data/logstash-input-proc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-proc'
-  s.version         = '0.1.1'
+  s.version         = '0.1.2'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This plugin is for reading the /proc of a linux filesystem"
   s.description = "Plugin is used with logstash"

data/quickstart.md

@@ -0,0 +1,45 @@
+# Logstash Plugin
+
+## To install for use
+```
+${LS_HOME}/bin/plugin install logstash-input-proc
+```
+
+## Example Config all features enabled
+```ruby
+input {
+    proc {
+        interval=>60
+        vmstats =>{ }
+        loadavg =>{ }
+        meminfo =>{ }
+        pidstats =>{ 
+            user => "root"
+        }
+        
+    }
+}
+
+output { 
+    stdout{ 
+        codec=>"rubydebug"
+    }
+}
+```
+##Example Minimal
+
+```ruby
+input {
+    proc {
+        interval=>60
+        meminfo =>{ }
+    }
+}
+
+output { 
+    stdout{ 
+        codec=>"rubydebug"
+    }
+}
+```
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-proc
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Edward Perry
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-20 00:00:00.000000000 Z
+date: 2015-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -82,15 +82,18 @@ files:
 - ElasticSearch/README.md
 - ElasticSearch/templates/loadavg.json
 - ElasticSearch/templates/meminfo.json
+- ElasticSearch/templates/pidstats.json
 - ElasticSearch/templates/vmstats.json
 - Gemfile
 - Kibana/READEME.md
 - Kibana/export.json
 - README.md
 - Rakefile
+- documentation.md
 - lib/logstash/inputs/proc.rb
 - logstash-input-proc.gemspec
 - logstash.conf
+- quickstart.md
 - spec/inputs/proc_spec.rb
 homepage: http://eperry.github.io/logstash-input-proc/
 licenses: