processwatch 3.0.8 → 3.0.9

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/CHANGELOG

@@ -0,0 +1,9 @@
+License: (MIT) Copyright (C) 2013 Process Watch Author Phil Chen.
+
+3.0.9 Aug 18 2013
+====
+* Added system utilization statistics for when a process issue is detected prior to correction
+
+3.0.8 July 19 2013
+====
+* Converted Process Watch a Ruby Gem

data/README.md

@@ -1,4 +1,4 @@
-# Processwatch
+# Process Watch
 
 License: (MIT) Copyright (C) 2013 Phil Chen.
 
@@ -14,6 +14,8 @@ This is useful for systems issues, automating troubleshooting, provisioning, sca
 
   Detecting if a process is a runaway if so execute some combination of notify and killing it
 
+  Email system utilization statistics in the state prior to correcting the runaway process or starting a dead one
+
 ## PREREQUISITES:
 
   Linux, Ruby, Cron (Or some other scheduler)
@@ -43,7 +45,8 @@ ruby setup.rb
 Configure the configuration files locate in /usr/local/processwatch/conf/
 
 Note you can have multiple services monitored for restart.
-Sample restart configuration file begins with prefix restart_ naming convention looks like restart_uniqueappname
+See example restart configuration file restart_ssh
+You can add more restart processes with the naming convention restart_uniqueappname (note prefix restart_)
 
 ## USING PROCESSWATCH:
 

data/lib/processwatch.rb

@@ -1,3 +1,4 @@
 require 'processwatch/version'
 require 'processwatch/pw_setup'
+require 'processwatch/pw_diagnostic.rb'
 require 'processwatch/pw_linux'

data/lib/processwatch/pw_diagnostic.rb

@@ -0,0 +1,312 @@
+#License: (MIT), Copyright (C) 2013 Process Watch Author Phil Chen.
+module Usagewatch
+  # Show the amount of total disk used in Gigabytes
+  def self.uw_diskused
+    @df = `df`
+    @parts = @df.split(" ").map { |s| s.to_i }
+    @sum = 0
+    for i in (9..@parts.size - 1).step(6) do
+      @sum += @parts[i]
+    end
+    @round = @sum.round(2)
+    @totaldiskused = ((@round/1024)/1024).round(2)
+  end
+
+  # Show the percentage of disk used.
+  def self.uw_diskused_perc
+    df = `df --total`
+    df.split(" ").last.to_f.round(2)
+  end
+
+  # Show the percentage of CPU used
+  def self.uw_cpuused
+    @proc0 = File.readlines('/proc/stat').grep(/^cpu /).first.split(" ")
+    sleep 1
+    @proc1 = File.readlines('/proc/stat').grep(/^cpu /).first.split(" ")
+
+    @proc0usagesum = @proc0[1].to_i + @proc0[2].to_i + @proc0[3].to_i
+    @proc1usagesum = @proc1[1].to_i + @proc1[2].to_i + @proc1[3].to_i
+    @procusage = @proc1usagesum - @proc0usagesum
+
+    @proc0total = 0
+    for i in (1..4) do
+      @proc0total += @proc0[i].to_i
+    end
+    @proc1total = 0
+    for i in (1..4) do
+      @proc1total += @proc1[i].to_i
+    end
+    @proctotal = (@proc1total - @proc0total)
+
+    @cpuusage = (@procusage.to_f / @proctotal.to_f)
+    @cpuusagepercentage = (100 * @cpuusage).to_f.round(2)
+  end
+
+  # return hash of top ten proccesses by cpu consumption
+  # example [["apache2", 12.0], ["passenger", 13.2]]
+  def self.uw_cputop
+    ps = `ps aux | awk '{print $11, $3}' | sort -k2nr  | head -n 10`
+    array = []
+    ps.each_line do |line|
+      line = line.chomp.split(" ")
+      array << [line.first.gsub(/[\[\]]/, ""), line.last]
+    end
+    array
+  end
+
+  # Show the number of TCP connections used
+  def self.uw_tcpused
+    if File.exists?("/proc/net/sockstat")
+      File.open("/proc/net/sockstat", "r") do |ipv4|
+        @sockstat = ipv4.read
+      end
+
+      @tcp4data = @sockstat.split
+      @tcp4count = @tcp4data[5]
+    end
+
+    if  File.exists?("/proc/net/sockstat6")
+      File.open("/proc/net/sockstat6", "r") do |ipv6|
+        @sockstat6 = ipv6.read
+
+      end
+
+      @tcp6data = @sockstat6.split
+      @tcp6count = @tcp6data[2]
+    end
+
+    @totaltcpused = @tcp4count.to_i + @tcp6count.to_i
+  end
+
+  # Show the number of UDP connections used
+  def self.uw_udpused
+    if File.exists?("/proc/net/sockstat")
+      File.open("/proc/net/sockstat", "r") do |ipv4|
+        @sockstat = ipv4.read
+      end
+
+      @udp4data = @sockstat.split
+      @udp4count = @udp4data[16]
+    end
+
+    if File.exists?("/proc/net/sockstat6")
+      File.open("/proc/net/sockstat6", "r") do |ipv6|
+        @sockstat6 = ipv6.read
+      end
+
+      @udp6data = @sockstat6.split
+      @udp6count = @udp6data[5]
+    end
+
+    @totaludpused = @udp4count.to_i + @udp6count.to_i
+  end
+
+  # Show the percentage of Active Memory used
+  def self.uw_memused
+    if File.exists?("/proc/meminfo")
+      File.open("/proc/meminfo", "r") do |file|
+        @result = file.read
+      end
+    end
+
+    @memstat = @result.split("\n").collect{|x| x.strip}
+    @memtotal = @memstat[0].gsub(/[^0-9]/, "")
+    @memactive = @memstat[5].gsub(/[^0-9]/, "")
+    @memactivecalc = (@memactive.to_f * 100) / @memtotal.to_f
+    @memusagepercentage = @memactivecalc.round
+  end
+
+  # return hash of top ten proccesses by mem consumption
+  # example [["apache2", 12.0], ["passenger", 13.2]]
+  def self.uw_memtop
+    ps = `ps aux | awk '{print $11, $4}' | sort -k2nr  | head -n 10`
+    array = []
+    ps.each_line do |line|
+      line = line.chomp.split(" ")
+      array << [line.first.gsub(/[\[\]]/, ""), line.last]
+    end
+    array
+  end
+
+  # Show the average system load of the past minute
+  def self.uw_load
+    if File.exists?("/proc/loadavg")
+      File.open("/proc/loadavg", "r") do |file|
+        @loaddata = file.read
+      end
+
+      @load = @loaddata.split(/ /).first.to_f
+    end
+  end
+
+  # Bandwidth Received Method
+  def self.bandrx
+
+    if File.exists?("/proc/net/dev")
+      File.open("/proc/net/dev", "r") do |file|
+        @result = file.read
+      end
+    end
+
+    @arrRows = @result.split("\n")
+
+    @arrEthLoRows = @arrRows.grep(/eth|lo/)
+
+    rowcount = (@arrEthLoRows.count - 1)
+
+    for i in (0..rowcount)
+      @arrEthLoRows[i] = @arrEthLoRows[i].gsub(/\s+/m, ' ').strip.split(" ")
+    end
+
+    @arrColumns = Array.new
+    for l in (0..rowcount)
+      @temp = Array.new
+      @temp[0] = @arrEthLoRows[l][1]
+      @temp[1] = @arrEthLoRows[l][9]
+      @arrColumns << @temp
+    end
+
+    columncount = (@arrColumns[0].count - 1)
+
+    @arrTotal = Array.new
+    for p in (0..columncount)
+      @arrTotal[p] = 0
+    end
+
+    for j in (0..columncount)
+      for k in (0..rowcount)
+        @arrTotal[j] = @arrColumns[k][j].to_i + @arrTotal[j]
+      end
+    end
+
+    @bandrxtx = @arrTotal
+  end
+
+  # Current Bandwidth Received Calculation in Mbit/s
+  def self.uw_bandrx
+
+    @new0 = self.bandrx
+    sleep 1
+    @new1 = self.bandrx
+
+    @bytesreceived = @new1[0].to_i - @new0[0].to_i
+    @bitsreceived = (@bytesreceived * 8)
+    @megabitsreceived = (@bitsreceived.to_f / 1024 / 1024).round(3)
+  end
+
+  # Bandwidth Transmitted Method
+  def self.bandtx
+
+    if File.exists?("/proc/net/dev")
+      File.open("/proc/net/dev", "r") do |file|
+        @result = file.read
+      end
+    end
+
+    @arrRows = @result.split("\n")
+
+    @arrEthLoRows = @arrRows.grep(/eth|lo/)
+
+    rowcount = (@arrEthLoRows.count - 1)
+
+    for i in (0..rowcount)
+      @arrEthLoRows[i] = @arrEthLoRows[i].gsub(/\s+/m, ' ').strip.split(" ")
+    end
+
+    @arrColumns = Array.new
+    for l in (0..rowcount)
+      @temp = Array.new
+      @temp[0] = @arrEthLoRows[l][1]
+      @temp[1] = @arrEthLoRows[l][9]
+      @arrColumns << @temp
+    end
+
+    columncount = (@arrColumns[0].count - 1)
+
+    @arrTotal = Array.new
+    for p in (0..columncount)
+      @arrTotal[p] = 0
+    end
+
+    for j in (0..columncount)
+      for k in (0..rowcount)
+        @arrTotal[j] = @arrColumns[k][j].to_i + @arrTotal[j]
+      end
+    end
+
+    @bandrxtx = @arrTotal
+  end
+
+  # Current Bandwidth Transmitted in Mbit/s
+  def self.uw_bandtx
+
+    @new0 = self.bandtx
+    sleep 1
+    @new1 = self.bandtx
+
+    @bytestransmitted = @new1[1].to_i - @new0[1].to_i
+    @bitstransmitted = (@bytestransmitted * 8)
+    @megabitstransmitted = (@bitstransmitted.to_f / 1024 / 1024).round(3)
+  end
+
+  # Disk Usage Method
+  def self.diskio
+
+    if File.exists?("/proc/diskstats")
+      File.open("/proc/diskstats", "r") do |file|
+        @result = file.read
+      end
+    end
+
+    @arrRows = @result.split("\n")
+
+    rowcount = (@arrRows.count - 1)
+
+    for i in (0..rowcount)
+      @arrRows[i] = @arrRows[i].gsub(/\s+/m, ' ').strip.split(" ")
+    end
+
+    @arrColumns = Array.new
+    for l in (0..rowcount)
+      @temp = Array.new
+      @temp[0] = @arrRows[l][3]
+      @temp[1] = @arrRows[l][7]
+      @arrColumns << @temp
+    end
+
+    columncount = (@arrColumns[0].count - 1)
+
+    @arrTotal = Array.new
+    for p in (0..columncount)
+      @arrTotal[p] = 0
+    end
+
+    for j in (0..columncount)
+      for k in (0..rowcount)
+        @arrTotal[j] = @arrColumns[k][j].to_i + @arrTotal[j]
+      end
+    end
+
+    @diskiorw= @arrTotal
+  end
+
+  # Current Disk Reads Completed
+  def self.uw_diskioreads
+
+    @new0 = self.diskio
+    sleep 1
+    @new1 = self.diskio
+
+    @diskreads = @new1[0].to_i - @new0[0].to_i
+  end
+
+  # Current Disk Writes Completed
+  def self.uw_diskiowrites
+
+    @new0 = self.diskio
+    sleep 1
+    @new1 = self.diskio
+
+    @diskwrites = @new1[1].to_i - @new0[1].to_i
+  end
+end

data/lib/processwatch/pw_linux.rb

@@ -15,6 +15,25 @@ module Processwatch
 
     list = ps_list.split(/\n/)
 
+    occurances = list.grep(/.*#$restart_process.*/)
+    if occurances.length == 0 && $restart_mail == "yes" && $restart_start == "yes"
+      usw =  Usagewatch
+diagnostic_msgstr = <<END_OF_MESSAGE
+ #{usw.uw_diskused} Total Gigabytes Disk Used
+ #{usw.uw_diskused_perc} Percentage of Gigabytes Used
+ #{usw.uw_cpuused}% CPU Used
+ #{usw.uw_tcpused} TCP Connections Used
+ #{usw.uw_udpused} UDP Connections Used
+ #{usw.uw_memused}% Active Memory Used
+ #{usw.uw_load} Average System Load Of The Past Minute
+ #{usw.uw_bandrx} Mbit/s Current Bandwidth Received
+ #{usw.uw_bandtx} Mbit/s Current Bandwidth Transmitted
+ #{usw.uw_diskioreads}/s Current Disk Reads Completed
+ #{usw.uw_diskiowrites}/s Current Disk Writes Completed
+ Top Ten Processes By CPU Consumption: #{usw.uw_cputop}
+ Top Ten Processes By Memory Consumption: #{usw.uw_memtop}
+END_OF_MESSAGE
+
 restart_msgstr = <<END_OF_MESSAGE
 From: #$restart_from <#$restart_from_email>
 To: #$restart_to <#$restart_to_email>
@@ -24,8 +43,34 @@ Process Watch has detected #{$restart_process} has died.
 
 We have started #{$restart_process} again.
 
+Below Is a Snapshot Of System Statistics Before The Process Was Started Again
+
+#{diagnostic_msgstr}
+
 -Process Watch
 
+END_OF_MESSAGE
+        system $restart_action
+          require 'net/smtp'
+          Net::SMTP.start("#$restart_smtp_host", "#$restart_smtp_port") do |smtp|
+          smtp.send_message(restart_msgstr, "#$restart_from_email", "#$restart_to_email")
+    end
+    elsif occurances.length == 0 && $restart_mail == "yes" && $restart_start == "no"
+      usw =  Usagewatch
+diagnostic_msgstr = <<END_OF_MESSAGE
+ #{usw.uw_diskused} Total Gigabytes Disk Used
+ #{usw.uw_diskused_perc} Percentage of Gigabytes Used
+ #{usw.uw_cpuused}% CPU Used
+ #{usw.uw_tcpused} TCP Connections Used
+ #{usw.uw_udpused} UDP Connections Used
+ #{usw.uw_memused}% Active Memory Used
+ #{usw.uw_load} Average System Load Of The Past Minute
+ #{usw.uw_bandrx} Mbit/s Current Bandwidth Received
+ #{usw.uw_bandtx} Mbit/s Current Bandwidth Transmitted
+ #{usw.uw_diskioreads}/s Current Disk Reads Completed
+ #{usw.uw_diskiowrites}/s Current Disk Writes Completed
+ Top Ten Processes By CPU Consumption: #{usw.uw_cputop}
+ Top Ten Processes By Memory Consumption: #{usw.uw_memtop}
 END_OF_MESSAGE
 
 dead_process_msgstr = <<END_OF_MESSAGE
@@ -35,18 +80,13 @@ Subject: Dead Process Detected On #{host}
 
 Process Watch has detected #{$restart_process} has died.
 
+Below Is A Snapshot Of Current System Statistics
+
+#{diagnostic_msgstr}
+
 -Process Watch
 
 END_OF_MESSAGE
-
-    occurances = list.grep(/.*#$restart_process.*/)
-    if occurances.length == 0 && $restart_mail == "yes" && $restart_start == "yes"
-      system $restart_action
-        require 'net/smtp'
-        Net::SMTP.start("#$restart_smtp_host", "#$restart_smtp_port") do |smtp|
-        smtp.send_message(restart_msgstr, "#$restart_from_email", "#$restart_to_email")
-    end
-    elsif occurances.length == 0 && $restart_mail == "yes" && $restart_start == "no"
       require 'net/smtp'
       Net::SMTP.start("#$restart_smtp_host", "#$restart_smtp_port") do |smtp|
       smtp.send_message(dead_process_msgstr, "#$restart_from_email", "#$restart_to_email")
@@ -74,26 +114,80 @@ end
       cpu_time = ($1.to_i*3600.to_i + $2.to_i*60.to_i + $3.to_i)
     next if cpu_time.to_i < $runaway_max_time.to_i
     begin
+        if $runaway_mail == "yes" && $runaway_kill == "yes"
+          usw =  Usagewatch
+diagnostic_msgstr = <<END_OF_MESSAGE
+ #{usw.uw_diskused} Total Gigabytes Disk Used
+ #{usw.uw_diskused_perc} Percentage of Gigabytes Used
+ #{usw.uw_cpuused}% CPU Used
+ #{usw.uw_tcpused} TCP Connections Used
+ #{usw.uw_udpused} UDP Connections Used
+ #{usw.uw_memused}% Active Memory Used
+ #{usw.uw_load} Average System Load Of The Past Minute
+ #{usw.uw_bandrx} Mbit/s Current Bandwidth Received
+ #{usw.uw_bandtx} Mbit/s Current Bandwidth Transmitted
+ #{usw.uw_diskioreads}/s Current Disk Reads Completed
+ #{usw.uw_diskiowrites}/s Current Disk Writes Completed
+ Top Ten Processes By CPU Consumption: #{usw.uw_cputop}
+ Top Ten Processes By Memory Consumption: #{usw.uw_memtop}
+END_OF_MESSAGE
 
-msgstr = <<END_OF_MESSAGE
+kill_msgstr = <<END_OF_MESSAGE
 From: #$runaway_from <#$runaway_from_email>
 To: #$runaway_to <#$runaway_to_email>
 Subject: Runaway Process Detected
 
+The Below Runaway Process Has Been Detected
+
 #{process[4]}
 
+It has been killed!
+
+Below Is A Snapshot Of The System Statistics Before It Was Killed
+
+#{diagnostic_msgstr}
 END_OF_MESSAGE
 
-        if $runaway_mail == "yes" && $runaway_kill == "yes"
           require 'net/smtp'
           Net::SMTP.start("#$runaway_smtp_host", "#$runaway_smtp_port") do |smtp|
-          smtp.send_message(msgstr, "#$runaway_from_email", "#$runaway_to_email")
+          smtp.send_message(kill_msgstr, "#$runaway_from_email", "#$runaway_to_email")
             system("/bin/kill" + " " + "-9" + " " + "#{process[2]}")
         end
         elsif $runaway_mail == "yes" && $runaway_kill == "no"
+          usw =  Usagewatch
+diagnostic_msgstr = <<END_OF_MESSAGE
+ #{usw.uw_diskused} Total Gigabytes Disk Used
+ #{usw.uw_diskused_perc} Percentage of Gigabytes Used
+ #{usw.uw_cpuused}% CPU Used
+ #{usw.uw_tcpused} TCP Connections Used
+ #{usw.uw_udpused} UDP Connections Used
+ #{usw.uw_memused}% Active Memory Used
+ #{usw.uw_load} Average System Load Of The Past Minute
+ #{usw.uw_bandrx} Mbit/s Current Bandwidth Received
+ #{usw.uw_bandtx} Mbit/s Current Bandwidth Transmitted
+ #{usw.uw_diskioreads}/s Current Disk Reads Completed
+ #{usw.uw_diskiowrites}/s Current Disk Writes Completed
+ Top Ten Processes By CPU Consumption: #{usw.uw_cputop}
+ Top Ten Processes By Memory Consumption: #{usw.uw_memtop}
+END_OF_MESSAGE
+
+no_kill_msgstr = <<END_OF_MESSAGE
+From: #$runaway_from <#$runaway_from_email>
+To: #$runaway_to <#$runaway_to_email>
+Subject: Runaway Process Detected
+
+The Below Runaway Process Has Been Detected
+
+#{process[4]}
+
+Below Is A Snapshot Of Current System Statistics
+
+#{diagnostic_msgstr}
+END_OF_MESSAGE
+
           require 'net/smtp'
           Net::SMTP.start("#$runaway_smtp_host", "#$runaway_smtp_port") do |smtp|
-          smtp.send_message(msgstr, "#$runaway_from_email", "#$runaway_to_email")
+          smtp.send_message(no_kill_msgstr, "#$runaway_from_email", "#$runaway_to_email")
         end
         elsif $runaway_mail == "no" && $runaway_kill == "yes"
           system("/bin/kill" + " " + "-9" + " " + "#{process[2]}")

data/lib/processwatch/version.rb

@@ -1,3 +1,3 @@
 module Processwatch
-  VERSION = "3.0.8"
+  VERSION = "3.0.9"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: processwatch
 version: !ruby/object:Gem::Version
-  version: 3.0.8
+  version: 3.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-20 00:00:00.000000000 Z
+date: 2013-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -51,11 +51,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .gitignore
+- CHANGELOG
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - lib/processwatch.rb
+- lib/processwatch/pw_diagnostic.rb
 - lib/processwatch/pw_linux.rb
 - lib/processwatch/pw_setup.rb
 - lib/processwatch/version.rb