continuent-monitors-nagios 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac732a7a2022300057c022f39230bda92f1122bb
-  data.tar.gz: 175e84d96fd7cf1c34971c316289610fd542490c
+  metadata.gz: 4a06f684ddc84caf2d0b1c859a33b5d34b5d2d63
+  data.tar.gz: a0bebc5b0902b9bbe71460e6335e757529b79c90
 SHA512:
-  metadata.gz: 82b76f8298a985a862a8b8d1db26d035a0f143dbbd1bfe4624139b0989bcec471e762b0e0c8b565da80e7ba19781c2c8e453c7272ffa31f909a8b15fff3ce435
-  data.tar.gz: 80950ef9e1f10fe09cbf8131dcfd95bdbcc079a334218929103b5e5b04abf3cdfea1e1ebdb081f7fbc813c65f438217c43057c5538a53f7673c6a6272fd17b9e
+  metadata.gz: 44f04b5c74c7409d300fdbed92c1e2bc010727549625801d859092693e9f7c49d75d7e927cc35ddfbef5ae17a7f0746c7dad7671a146ecec486ef4a1c760ce19
+  data.tar.gz: 9f713e08e75e95625284cf086e41ddf49469b8fd3b62cfa4bb89c7b17b1d9505db15ac110b55cb46b3a7df218cbf4c693a8029c1dadde1a40240d42f7ec55645

data/bin/check_tungsten.sh

@@ -0,0 +1,576 @@
+#!/bin/bash
+
+#TODO
+#Work out the cluster names in a Composite DS
+#determine the individulal services in a replicator so we can print out better output (status on each)
+#Remove host logging - stop duplicate emails across multi hosts?
+
+HOST=`hostname`
+
+#Start Configuration Options. - These can be overridden by command line options or from $CONTINUENT_ROOT/share/check_tungsten.cfg
+CONNECTOR=0      					#If this host is running a connector set to 1 otherwise 0
+CLUSTER=0      						#If this host is running a cluster set to 1 otherwise 0
+REPLICATOR=0   						#If this host is running a replicator  set to 1 otherwise 0
+REPLICATOR_PORT=10000      			#Replicator Port
+REPLICATOR_HOME=/opt/continuent/    #Home dir for Replicator
+SERVICES=''                         #Name of the individual clusters in a composite DS
+EMAIL=''							#Set email address here or pass via the email= command line option
+DISK=0                              #Check Disk space
+CHECK_ELB=0                         #Enable check for ELB socket check
+
+SUBJECT="Error : Problems exist with the Tungsten Services on $HOST"
+LOCK_TIMEOUT=180       				# Only send a Email every x minutes for a specific 
+                      				# problem, stop spamming from the script
+LAG=60                				# Slave lag to report on
+CONNECTOR_TIMEOUT=10  				# No of seconds to wait for a connector response
+DISK_WARNING=80       				# % full to send a warning
+SENDMAILBIN=/usr/sbin/sendmail
+#End Configuration Options
+
+SENDMAIL=0
+DEBUG=0
+LOG=/opt/continuent/share/check_tungsten.log
+LOCK_DIR=/opt/continuent/share/tungsten_locks
+
+function float_cond()
+{
+    local cond=0
+    if [[ $# -gt 0 ]]; then
+        cond=$(echo "$*" | bc -q 2>&1)
+        if [[ $? -ne 0 ]]; then
+            echo "Error: $cond"
+            exit 1
+        fi
+        if [[ -z "$cond" ]]; then cond=0; fi
+        if [[ "$cond" != 0  &&  "$cond" != 1 ]]; then cond=0; fi
+    fi
+    local stat=$((cond == 0))
+    return $stat
+}
+
+info ()
+{
+	if [ $DEBUG == 1 ]; then echo "INFO    : $1"; fi
+}
+error ()
+{
+	if [ $DEBUG == 1 ]; then echo "ERROR   : $1"; fi
+}
+severe ()
+{
+	echo "SEVERE  : $1"
+	exit 1
+}
+getSetting ()
+{
+	CFG=$CONTINUENT_ROOT/conf/tungsten.cfg
+	if [ ! -f $CFG ]
+	then
+		severe "Unable to find $CFG"
+	fi
+	getSettingValue=""
+	getSettingValue=$(grep "\"$1\"" $CFG| cut -d ':' -f2 | head -1|sed 's/,//g'|sed 's/"//g'|sed 's/ //g')
+	if [ -z $getSettingValue ]
+	then
+		severe "Unable to find $1 in $CFG"
+	fi
+	if [ "$getSettingValue" == '' ]
+	then
+		severe "Unable to find $1 in $CFG"
+	fi
+	echo "$getSettingValue"
+}
+
+
+# Load any continuent variables 
+
+if [ -z $CONTINUENT_ROOT ]
+then
+	[ -f "$HOME/.bash_profile" ] && . "$HOME/.bash_profile"
+	[ -f "$HOME/.profile" ] && . "$HOME/.profile"
+fi
+
+function sOpt()
+{
+	$1=1
+	info "$1 switched on via command line"
+}
+
+function mOpt()
+{
+	
+	for i in $(echo $2 | tr "=" "\n")
+	do
+		if [ $i != '$3' ]
+		then
+			$1=$i
+		fi
+	done
+}
+
+#Parse the command line options
+
+for arg in "$@"
+do
+    case "$arg" in
+    -v)    				DEBUG=1
+      					info "Debug mode set"
+            			;;
+    -vv)    			DEBUG=1
+      					info "INFO   : Extended Debug mode set"
+						set -x
+        				;;
+    cluster)    		CLUSTER=1
+            			info "CLUSTER switched on via command line" 
+            			;;
+    connector)  		CONNECTOR=1
+            			info "CONNECTOR switched on via command line" 
+            			;;
+    replicator)   		REPLICATOR=1
+            			info "REPLICATOR switched on via command line"
+            			;;
+    check_elb)   		CHECK_ELB=1
+            			info "CHECK_ELB switched on via command line"
+            			;;
+    replicator_port*)   for i in $(echo $arg | tr "=" "\n")
+						do
+							if [ $i != 'replicator_port' ]
+							then
+				  				REPLICATOR_PORT=$i
+							fi
+						done
+
+            			info "REPLICATOR_PORT - $REPLICATOR_PORT - switched on via command line" 
+            			;;
+    replicator_home*)   for i in $(echo $arg | tr "=" "\n")
+						do
+							if [ $i != 'replicator_home' ]
+							then
+				  				REPLICATOR_HOME=$i
+							fi
+						done
+
+            			info "REPLICATOR_HOME - $REPLICATOR_HOME - switched on via command line" 
+            			;;
+    services*)   		for i in $(echo $arg | tr "=" "\n")
+						do
+							if [ $i != 'services' ]
+							then
+				  				SERVICES=$i
+							fi
+						done
+            			info "SERVICES $SERVICES passed via the command line" 
+            			;;
+    email*)   			for i in $(echo $arg | tr "=" "\n")
+						do
+							if [ $i != 'email' ]
+							then
+				  				EMAIL=$i
+							fi
+						done
+            			info "EMAIL $EMAIL passed via the command line" 
+            			;;
+    config*)   			for i in $(echo $arg | tr "=" "\n")
+                		do
+                			if [ $i != 'config' ]
+                			then
+                    			FILE=$i
+                			fi
+                		done
+            			info "Config File $FILE passed via the command line"
+            			;;
+    disk)   			DISK=1
+            			info "DISK switched on via command line"
+            			;;
+    *)
+            			echo "Unknown command line option passed $arg"
+            			echo "Valid options are -v,cluster,connector,replicator,replicator_port=??,services=??,email=??,config=??"
+            			exit 1
+    esac
+
+    
+done
+
+
+if [ $CLUSTER == 1 ] || [ $CONNECTOR == 1 ]
+then
+	if [ -z $CONTINUENT_ROOT ]
+	then
+		severe "$CONTINUENT_ROOT is not set - unable to continue"
+	fi
+	if [ ! -f $CONTINUENT_ROOT/share/env.sh ]
+	then
+		severe "Unable to find env.sh in $CONTINUENT_ROOT/share"
+	fi
+
+. "$CONTINUENT_ROOT/share/env.sh"
+	
+#Load any default settings from $CONTINUENT_ROOT/share/check_tungsten.cfg
+	CFG=$CONTINUENT_ROOT/share/check_tungsten.cfg
+	
+	if [  -f $CFG ]
+	then
+	     info "Loading settings from $CFG"
+	     . "$CFG"
+	fi
+	if [ -z "$MYSQL" ]
+	then
+		MYSQL=`which mysql 2>/dev/null`
+	      
+		if [ "$MYSQL" == "" ]
+	    then
+			severe " Unable to the mysql command line program"
+	    fi
+	fi
+fi
+
+#If a file is passed from the command line load any variables from there
+if [ ! -z $FILE ]
+then
+    if [ ! -f $FILE ]
+    then
+        severe "The file specified in the command line $FILE does not exist"
+    fi
+
+    info "Loading settings from $FILE"
+    . "$FILE"
+fi
+
+#Parameter and host validation
+
+BC=`which bc 2>/dev/null`
+      
+if [ "$BC" == "" ]
+then
+   severe " Unable to find the command bc - please install"
+fi
+      
+
+if [ "$EMAIL" == "" ]
+then
+	severe " email must be specified"
+fi
+
+if [[  "$CONNECTOR" == 0  &&  "$CLUSTER" == 0  &&  "$REPLICATOR" == 0  ]]
+then
+	severe " No option specified, select either connector, cluster or replicator"
+fi
+
+if [ -d $LOCK_DIR ]
+then
+	if [ ! -w $LOCK_DIR ]
+	then
+		severe " The locks dir $LOCK_DIR is not writable"
+	fi
+else
+	info "Creating locks dir" 
+	mkdir $LOCK_DIR
+fi
+
+if [ -z "$MAILPROG" ]
+  then
+     MAILPROG=`which mail 2>/dev/null`
+
+  	if [ "$MAILPROG" == "" ]
+      then
+  		severe " Unable to find a mail program"
+      fi
+  fi
+
+if [ -z "$SENDMAILBIN" ]
+  then
+     SENDMAILBIN=`which sendmail 2>/dev/null`
+
+  	if [ "$SENDMAILBIN" == "" ]
+      then
+  		severe " Unable to find a sendmail program"
+      fi
+fi
+
+if [ -f $LOG ]
+then
+   rm $LOG
+fi
+
+#Expire old Locks
+info "Deleting Locks older than $LOCK_TIMEOUT min" 
+find $LOCK_DIR/* -type f -mmin +$LOCK_TIMEOUT -delete 2> /dev/null
+
+#Check the connector status
+if [ $CONNECTOR == 1 ]
+then
+	connector_ok_to_allow_elb=0
+	info "Running Connector Tests"
+	CONN=$($CONTINUENT_ROOT/tungsten/cluster-home/bin/check_tungsten_services -c| grep -v OK | wc -l)
+	if [ $CONN -ne  0 ]
+	then
+	   error " Connector is not running" 
+	   echo "Connector is not running on $HOST - Investigate" >> $LOG
+	   if [ ! -f $LOCK_DIR/con_running.lck ]
+	   then
+	      SENDMAIL=1
+	      touch $LOCK_DIR/con_running.lck
+	   else
+	      info "Not sending Email lock file exists" 
+	   fi
+	else 
+		info "Connector is running OK" 
+
+		TIMEOUT=`which timeout 2>/dev/null`
+      
+		if [ "$TIMEOUT" == "" ]
+	    then
+			info "timeout command not found - unable to check if the connector is responding"
+		else
+			info "Checking Connector is responding to queries"
+			CON_USER=$(getSetting connector_user)
+			CON_PW=$(getSetting connector_password)
+			CON_PORT=$(getSetting connector_listen_port)
+			CHECK=$(timeout -s HUP $CONNECTOR_TIMEOUT $MYSQL -P$CON_PORT -u $CON_USER -p$CON_PW -h $HOSTNAME --skip-column-names  -Be"select 'ALIVE'")
+			if [ "$CHECK" != 'ALIVE' ]
+			then
+				error 'Unable to connect to connector'
+				echo "Connector is not responding on $HOST - Investigate" >> $LOG
+				connector_ok=0
+				if [ ! -f $LOCK_DIR/con_responding.lck ]
+				   then
+				    SENDMAIL=1
+				    touch $LOCK_DIR/con_responding.lck
+				   else
+				      info "Not sending Email lock file exists" 
+				fi
+			else
+				info 'Connector is alive'
+				connector_ok_to_allow_elb=1
+			fi
+		fi
+	fi
+	
+	if [ $CHECK_ELB == 1 ]
+	then
+		if [ -f /etc/xinetd.d/disabled/connectorchk ]  && [ $connector_ok_to_allow_elb == 1 ]
+		then
+
+				sudo mv /etc/xinetd.d/disabled/connectorchk /etc/xinetd.d/
+				sudo service xinetd reload
+		fi
+	fi
+fi
+
+#Check the cluster Status
+if [ $CLUSTER == 1 ]
+then
+  #Check the processes are running
+  info "Running Cluster Tests"
+  REPL=$($CONTINUENT_ROOT/tungsten/cluster-home/bin/check_tungsten_services -r| grep -v OK | wc -l)
+  if [ $REPL -ne  0 ]
+  then
+    error " Replicator or Manager in cluster is not running"
+      echo "Replicator or Manager in cluster is not running on $HOST - Investigate" >> $LOG
+      if [ ! -f $LOCK_DIR/rep_running.lck ]
+      then
+         SENDMAIL=1
+         touch $LOCK_DIR/rep_running.lck
+      else
+        info "Not sending Email lock file exists" 
+      fi
+     
+  else
+    info "Replicator and Manager in cluster are running OK" 
+  fi
+
+  #Check the processes are online
+	if [ "$SERVICES" == "" ]  
+	then
+		  ONLINE=$($CONTINUENT_ROOT/tungsten/cluster-home/bin/check_tungsten_online | grep -v OK | wc -l)  
+		  if [ $ONLINE -ne  0 ]
+		  then
+		    error "Services are not online" 
+		      echo "Cluster Replicator processes are not online on $HOST - Investigate" >> $LOG
+		      if [ ! -f $LOCK_DIR/rep_online.lck ]
+		      then
+		         SENDMAIL=1
+		         touch $LOCK_DIR/rep_online.lck
+		      else
+		        info "Not sending Email lock file exists" 
+		      fi
+		
+		  else
+		    info "Services are online" 
+		  fi
+	else
+		services=$(echo "$SERVICES" | sed 's/,/ /g')
+		for s in $services
+		do
+			ONLINE=$($CONTINUENT_ROOT/tungsten/cluster-home/bin/check_tungsten_online -s $s | grep -v OK | wc -l)  
+			if [ $ONLINE -ne  0 ]
+			then
+		    	error "Services are not online @ $s" 
+		      echo "Cluster Replicator processes are not online on $HOST - Investigate" >> $LOG
+		      if [ ! -f $LOCK_DIR/rep_online.lck ]
+		      then
+		         SENDMAIL=1
+		         touch $LOCK_DIR/rep_online.lck
+		      else
+		        info "Not sending Email lock file exists" 
+		      fi
+		
+			else
+		    info "Services are online @ $s" 
+			fi
+		done
+	fi
+
+  #Check for replicator latency
+  ONLINE=$($CONTINUENT_ROOT/tungsten/cluster-home/bin/check_tungsten_latency -w $LAG -c $LAG | grep -v OK | wc -l)  
+  if [ $ONLINE -ne  0 ]
+  then
+    error "Services are Lagging" 
+      echo "Cluster Replicator processes are lagging on $HOST - Investigate" >> $LOG
+      if [ ! -f $LOCK_DIR/rep_lag.lck ]
+      then
+         SENDMAIL=1
+         touch $LOCK_DIR/rep_lag.lck
+      else
+        info "Not sending Email lock file exists" 
+      fi
+
+  else
+    info "Cluster Replicator is keeping up" 
+  fi
+fi
+
+#Check the Replicator 
+if [ $REPLICATOR == 1 ]
+then
+	if [ ! -f $REPLICATOR_HOME/tungsten/tungsten-replicator/bin/trepctl ]
+	then
+		severe "trepctl not found in $REPLICATOR_HOME/tungsten/tungsten-replicator/bin/ "
+	fi
+	
+	AVAILABLE=$($REPLICATOR_HOME/tungsten/tungsten-replicator/bin/trepctl -port $REPLICATOR_PORT services | grep "Connection failed" | wc -l)
+	if [ $AVAILABLE -gt 0 ]
+	then
+		error "Replicator process is not running on $REPLICATOR_PORT"
+		echo "Replicator processes is not running  on $HOST:$REPLICATOR_PORT - Investigate" >> $LOG
+		    if [ ! -f $LOCK_DIR/tr_rep_running.lck ]
+		      then
+		         SENDMAIL=1
+		        touch $LOCK_DIR/tr_rep_running.lck
+		      else
+		        info "Not sending Email lock file exists" 
+		      fi
+		
+	 else
+		    info "TR Replicator is running" 
+	  fi
+
+
+	ONLINE=$($REPLICATOR_HOME/tungsten/tungsten-replicator/bin/trepctl -port $REPLICATOR_PORT services| grep state | grep -v ONLINE | wc -l)
+	if [ $ONLINE -gt 0 ]
+	then
+		error "Replicator is down" 
+	    echo "Replicator processes is not ONLINE  on $HOST - Investigate" >> $LOG
+	      if [ ! -f $LOCK_DIR/tr_rep_online.lck ]
+	      then
+	         SENDMAIL=1
+	         touch $LOCK_DIR/tr_rep_online.lck
+	      else
+	        info "Not sending Email lock file exists" 
+	      fi
+	
+	  else
+	    info "TR Replicator is online" 
+	  fi
+	
+	#Check for latency
+	LATENCY_LIST=$($REPLICATOR_HOME/tungsten/tungsten-replicator/bin/trepctl -port $REPLICATOR_PORT services|grep appliedLatency|cut -d ':' -f2)
+    
+    for LATENCY in $LATENCY_LIST
+    do
+	    if float_cond "$LATENCY > $LAG"; then
+	    error "Replicator is lagging" 
+		    echo "Replicator processes is behind  on $HOST - Investigate" >> $LOG
+		    if [ ! -f $LOCK_DIR/tr_rep_lag.lck ]
+		    then
+		        SENDMAIL=1
+		        touch $LOCK_DIR/tr_rep_lag.lck
+		    else
+		        info "Not sending Email lock file exists" 
+		    fi
+		else
+			    info "Replicator latency ok"    
+	    fi
+	done
+
+ 
+fi
+
+#Check the disk space
+if [ $DISK == 1 ]
+then
+	
+	df -HP | grep -vE '^Filesystem|tmpfs|cdrom' | awk '{ print $5 " " $1 }' | while read output;
+	do
+	usep=$(echo $output | awk '{ print $1}' | cut -d'%' -f1  )
+	partition=$(echo $output | awk '{ print $2 }' )
+	if [ $usep -ge $DISK_WARNING ]; then
+		error "Running out of disk space on $partition"
+	    echo "Running out for disk space on $HOST $partition - Investigate" >> $LOG
+	    if [ ! -f $LOCK_DIR/disk.lck ]
+	    then
+	        SENDMAIL=1
+	        touch $LOCK_DIR/disk.lck
+	    else
+	        info "Not sending Email lock file exists" 
+	    fi
+	  fi
+ done
+	
+fi
+		  
+if [ $SENDMAIL == 1 ]
+then
+    if [ $DEBUG == 1 ]
+    then 
+        info "Sending Email to $EMAIL"
+        info "Subject $SUBJECT"
+        cat $LOG
+    fi
+
+	if [ $CLUSTER == 1 ] || [ $CONNECTOR == 1 ]
+	then
+		manager_running=$($CONTINUENT_ROOT/tungsten/tungsten-manager/bin/manager status | grep "PID" | wc -l) 
+  		if [ $manager_running -eq 1 ]; then
+			info "Adding cctrl output to email" 
+			echo >> $LOG
+			echo "OUTPUT FROM cctrl ls on $HOST" >> $LOG
+			echo '--------------------------------------------------' >> $LOG
+			echo 'ls' | $CONTINUENT_ROOT/tungsten/tungsten-manager/bin/cctrl -expert >> $LOG
+			echo '--------------------------------------------------' >> $LOG
+		else
+			info 'Manager not running skipping cctrl output'
+			echo "Manager not running unable to gather cctrl output" >> $LOG
+		fi
+		
+	fi
+	if [ $REPLICATOR == 1 ] 
+	then
+		if [ -f $REPLICATOR_HOME/tungsten/tungsten-replicator/bin/trepctl ]
+		then
+				info "Adding trepctl output to email" 
+				echo "OUTPUT FROM trepctl -port $REPLICATOR_PORT status on $HOST" >> $LOG
+				echo '--------------------------------------------------' >> $LOG
+
+                       
+				$REPLICATOR_HOME/tungsten/tungsten-replicator/bin/trepctl -port $REPLICATOR_PORT services >> $LOG
+				echo '--------------------------------------------------' >> $LOG
+		else
+				info 'trepctl not found'
+				echo "trepctl not found at $REPLICATOR_HOME/tungsten/tungsten-replicator/bin/trepctl unable to query for output" >> $LOG
+		fi
+	fi
+    $MAILPROG -s "$SUBJECT" "$EMAIL" < $LOG
+fi
+
+

data/bin/check_tungsten_backups

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+#
+# The script should be run as the tungsten user to ensure the
+# environment is set correctly.  Export the environment variables before
+# calling the script to modify behavior
+#
+
+require "#{File.dirname(__FILE__)}/../lib/ruby/tungsten"
+
+class CheckTungstenBackups
+  include TungstenScript
+  
+  def main
+    status = TI.status()
+    unless status.coordinator() == TI.hostname()
+      nagios_ok("Not running check because this node is not the coordinator")
+    end
+    seconds_since_epoch = TU.cmd_result("date +%s").to_i()
+    most_recent_backup = nil
+    
+    status.datasources().each{
+      |ds|
+      begin
+        TU.ssh_result("stat -c\"%n %Y\" #{TI.trepctl_property(status.name(), 'replicator.storage.agent.fs.directory')}/store*.properties 2>/dev/null", ds, TI.user()).split("\n").each{
+          |line|
+          stored_backup=line.split(" ")
+          stored_backup[1] = stored_backup[1].to_i()
+          
+          if most_recent_backup == nil || stored_backup[1] > most_recent_backup[:seconds]
+            most_recent_backup = {
+              :hostname => ds,
+              :filename => stored_backup[0],
+              :seconds => stored_backup[1]
+            }
+          end
+        }
+      rescue CommandError
+      end
+    }
+    
+    if most_recent_backup == nil
+      nagios_critical("Unable to find a backup on any datasource")
+    end
+    
+    age = seconds_since_epoch-most_recent_backup[:seconds]
+    if age > @options[:max_backup_age]
+      nagios_critical("#{most_recent_backup[:hostname]}:#{most_recent_backup[:filename]} [#{age}s] is older than #{@options[:max_backup_age]}s")
+    else
+      nagios_ok("The most recent backup is #{most_recent_backup[:hostname]}:#{most_recent_backup[:filename]} [#{age}s]")
+    end
+  end
+  
+  def configure
+    super()
+    
+    description("Check all local datasources to make sure one of them has a backup younger than the max allowed age")
+    add_option(:max_backup_age, {
+      :on => "--max-backup-age String",
+      :help => "Maximum allowed age in seconds of a backup on any machine",
+      :parse => method(:parse_integer_option),
+      :default => 86400
+    })
+  end
+  
+  def script_name
+    "check_tungsten_backups"
+  end
+  
+  self.new().run()
+end

data/bin/check_tungsten_latency

@@ -0,0 +1,172 @@
+#!/bin/bash
+#
+# Simple Bash Script To Check Tungsten Latency
+# Nagios Plugin For NRPE
+#
+# This script accepts two arguments, {{-w}} and {{-c}}.  The {{-w}} flag is 
+# the level at which a warning should be returned.  {{-c}} sets the level for 
+# a critical return value.  The script uses the maximum latency of any slave 
+# to determine the return value.
+#
+OK_STATE=0
+WARNING_STATE=1
+CRITICAL_STATE=2
+THOME=`dirname $0`
+
+function display_help()
+{
+  echo "Usage: ./check_tungsten_latency -w warning_level -c critical_level [-h]"
+  echo " -w                     Throw a warning alert if the maximum latency"
+  echo "                          is above this level"
+  echo " -c                     Throw a critical alert if the maximum latency"
+  echo "                          is above this level"
+  echo " --perfdata             Display performance data of the latency"
+  echo " --perslave-perfdata    Show performance latency values of each slave."
+  echo "                          If this is not set the maximum latency will be"
+  echo "                          displayed in the performace data"
+  echo " -h                     Display this message"
+  exit 0
+}
+
+# We will use this to make some floating point comparisons
+function float_cond()
+{
+    local cond=0
+    if [[ $# -gt 0 ]]; then
+        cond=$(echo "$*" | bc -q 2>&1)
+        if [[ $? -ne 0 ]]; then
+            echo "Error: $cond"
+            exit 1
+        fi
+        if [[ -z "$cond" ]]; then cond=0; fi
+        if [[ "$cond" != 0  &&  "$cond" != 1 ]]; then cond=0; fi
+    fi
+    local stat=$((cond == 0))
+    return $stat
+}
+
+warning_level=0
+critical_level=0
+perfdata="false"
+performance_data_default_glue=""
+performance_data_suffix=""
+perfdata_allslaves=""
+
+for arg
+do
+  delim=""
+  case "$arg" in
+  #translate --gnu-long-options to -g (short options)
+    --perfdata) args="${args}-p ";;
+    --perslave-perfdata) args="${args}-s ";;
+    #pass through anything else
+    *) [[ "${arg:0:1}" == "-" ]] || delim="\""
+      args="${args}${delim}${arg}${delim} ";;
+  esac
+done
+ 
+#Reset the positional parameters to the short options
+eval set -- $args
+
+while getopts "w:c:h:nps" Option
+do
+  case $Option in
+    w	)
+  	warning_level=$OPTARG
+  	;;
+  	c )
+  	critical_level=$OPTARG
+    ;;
+    h )
+    display_help
+    ;;
+    p )
+    perfdata="true"
+    ;;
+    s   )
+    perfdata_allslaves="true"
+    ;;
+  esac
+done
+if float_cond "$warning_level == 0"; then
+  echo "Error: warning_level has not been set"
+  echo ""
+  display_help
+fi
+
+if float_cond "$critical_level == 0"; then
+  echo "Error: critical_level has not been set"
+  echo ""
+  display_help
+fi
+
+if [ "$perfdata" == "true" ]; then
+  performance_data_default_glue=" "
+  performance_data_suffix=";$warning_level;$critical_level;;"
+fi
+
+error_message=""
+error_messaage_glue=""
+performance_data_glue=""
+performance_data="| "
+max_latency=0
+
+manager_running=`${THOME}/../../tungsten-manager/bin/manager status | grep "PID" | wc -l`
+# Check the manager status
+if [ $manager_running -eq 0 ]; then
+  echo "CRITICAL: Manager is not running"
+  exit $CRITICAL_STATE
+fi
+
+latency_values=`echo "ls -l" | ${THOME}/../../tungsten-manager/bin/cctrl | grep -E "REPLICATOR\(role=[relay|slave]|appliedLatency" | tr -d "| "`
+
+current_slave=""
+for line in $latency_values
+do
+  if [[ $current_slave == "" ]]
+  then
+    current_slave=`echo $line | grep "REPLICATOR" | cut -f 1 -d ":"`
+  else
+    applied_latency=`echo $line | grep "appliedLatency" | cut -f 2 -d ":"`
+    
+    if float_cond "$applied_latency > $max_latency"; then
+      max_latency=$applied_latency
+    fi
+    
+    if float_cond "$applied_latency > $warning_level"; then
+      error_message="$error_message$error_message_glue$current_slave=$applied_latency""s"
+      error_message_glue=", "
+    fi
+    
+    if float_cond "$applied_latency == -1"; then
+      error_message="$error_message$error_message_glue$current_slave is missing latency information"
+      error_message_glue=", "
+    fi
+
+    performance_data="$performance_data$performance_data_glue$current_slave=$applied_latency$performance_data_suffix"
+    performance_data_glue="$performance_data_default_glue"
+    current_slave=""
+  fi
+done
+
+if [ "$perfdata_allslaves" != "true" ]; then
+	 performance_data="|  max_latency=${max_latency}$performance_data_suffix"
+fi
+
+if [ "$perfdata" == "false" ]; then
+  performance_data=""
+fi
+
+if float_cond "$max_latency > $critical_level"; then
+  echo "CRITICAL: $error_message $performance_data" 
+  exit $CRITICAL_STATE
+fi
+
+if [[ $error_message != "" ]]; then
+  echo "WARNING: $error_message  $performance_data" 
+  exit $WARNING_STATE
+fi
+
+  echo "OK: All slaves are running normally (max_latency=${max_latency}) $performance_data " 
+
+exit $OK_STATE

data/bin/check_tungsten_online

@@ -0,0 +1,105 @@
+#!/bin/bash
+# Copyright (C) 2014 Continuent, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.  You may obtain
+# a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Initial developer(s): Jeff Mace
+# Contributor(s):
+#
+# Simple Bash Script To Check Tungsten Services
+# Nagios Plugin For NRPE
+#
+# This script does not accept any arguments.  It will return a warning if any 
+# of Tungsten resources is not in an {{ONLINE}} state.  It uses the output of 
+# the {{ls resources}} command to determine the current state.
+#
+OK_STATE=0
+WARNING_STATE=1
+CRITICAL_STATE=2
+THOME=`dirname $0`
+
+error_message=""
+error_messaage_glue=""
+offline_count=0
+dataservice=""
+skip_shun=0
+
+function display_help()
+{
+  echo "Usage: ./check_tungsten_online -s dataservice [-h]"
+  echo " -s                     The data service you would like to check"
+  echo " -h                     Display this message"
+  echo " -n                     Skip Shunned Services"
+  exit 0
+}
+
+while getopts "s:h:n" Option
+do
+  case $Option in
+    h )
+    display_help
+    ;;
+    s   )
+    dataservice=$OPTARG
+    ;;
+    n   )
+    skip_shun=1
+    ;;
+  esac
+done
+
+manager_running=`${THOME}/../../tungsten-manager/bin/manager status | grep "PID" | wc -l`
+# Check the manager status
+if [ $manager_running -eq 0 ]; then
+  echo "CRITICAL: Manager is not running"
+  exit $CRITICAL_STATE
+fi
+
+if [ "$dataservice" == "" ]; then
+    offline_services=`echo "ls resources" | ${THOME}/../../tungsten-manager/bin/cctrl | grep \| | grep : | grep -v ONLINE | tr -d "| " | cut -f 1,2 -d ":"`
+else
+    offline_services=`echo "use $dataservice; ls" | ${THOME}/../../tungsten-manager/bin/cctrl -multi | grep "(\(composite \)\?master\|(\(composite \)\?slave\|(relay" | grep -v ONLINE | tr -d "|" | cut -f 1 -d "("`
+fi
+
+for offline_service in $offline_services
+do
+  offline_count=$(($offline_count+1))
+  error_message="$error_message$error_message_glue$offline_service"
+  error_message_glue=", "
+done
+
+if [ $offline_count -gt 0 ]
+then
+  echo "CRITICAL: $error_message are not ONLINE"
+  exit $CRITICAL_STATE
+fi
+
+
+if [ $skip_shun -eq 0 ]
+then
+    if [ "$dataservice" == "" ]; then
+        shunned=`echo "ls" | ${THOME}/../../tungsten-manager/bin/cctrl | grep 'SHUNNED' | wc -l`
+    else
+        shunned=`echo "use $dataservice; ls" | ${THOME}/../../tungsten-manager/bin/cctrl -multi  | grep 'SHUNNED' | wc -l`
+    fi
+
+    if [ $shunned -gt 0 ]
+    then
+      echo "CRITICAL: Dataservices are shunned"
+      exit $CRITICAL_STATE
+    fi
+fi
+
+
+echo "OK: All services are online"
+exit $OK_STATE

data/bin/check_tungsten_policy

@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Simple Bash Script To Check Tungsten Policy
+# Nagios Plugin For NRPE
+#
+# This script does not accept any arguments.  It will return error if the
+# cluster is in maintenance mode
+#
+OK_STATE=0
+WARNING_STATE=1
+CRITICAL_STATE=2
+THOME=`dirname $0`
+
+error_message=""
+error_messaage_glue=""
+offline_count=0
+dataservice=""
+
+function display_help()
+{
+  echo "Usage: ./check_tungsten_policy -s dataservice [-h]"
+  echo " -s                     The data service you would like to check"
+  echo " -h                     Display this message"
+  exit 0
+}
+
+while getopts "s:h" Option
+do
+  case $Option in
+    h )
+    display_help
+    ;;
+    s   )
+    dataservice=$OPTARG
+    ;;
+  esac
+done
+
+manager_running=`${THOME}/../../tungsten-manager/bin/manager status | grep "PID" | wc -l`
+# Check the manager status
+if [ $manager_running -eq 0 ]; then
+  echo "CRITICAL: Manager is not running"
+  exit $CRITICAL_STATE
+fi
+
+if [ "$dataservice" == "" ]; then
+    maint_mode=`echo "ls " | ${THOME}/../../tungsten-manager/bin/cctrl | grep MAINTENANCE | wc -l`
+else
+    maint_mode=`echo "use $dataservice; ls " | ${THOME}/../../tungsten-manager/bin/cctrl | grep MAINTENANCE | wc -l`
+fi
+
+
+if [ $maint_mode -gt 0 ]
+then
+  echo "CRITICAL: Cluster is in Maintenance mode"
+  exit $CRITICAL_STATE
+fi
+
+
+echo "OK: Cluster is in Automatic Mode"
+exit $OK_STATE

data/bin/check_tungsten_progress

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Simple Bash Script To Check Tungsten Progress
+# Nagios Plugin For NRPE
+#
+# This script accepts three arguments, {{-w}}, {{-c}} and {{-t}}.  The {{-w}} flag 
+# is the level at which a warning should be returned.  {{-c}} sets the level for 
+# a critical return value.  The {{-t}} flag sets the amount of time used to wait 
+# between monitoring the progress of the cluster.  The script uses the difference 
+# between the values to determine if a warning or critical alert should be 
+# issued
+#
+OK_STATE=0
+WARNING_STATE=1
+CRITICAL_STATE=2
+THOME=`dirname $0`
+
+function display_help()
+{
+  echo "Usage: ./check_tungsten_progress -t time [-h]"
+  echo " -t     The number of seconds to wait when monitoring progress"
+  echo " -h     Display this message"
+  exit 0
+}
+
+# We will use this to make some floating point comparisons
+function float_cond()
+{
+    local cond=0
+    if [[ $# -gt 0 ]]; then
+        cond=$(echo "$*" | bc -q 2>&1)
+        if [[ $? -ne 0 ]]; then
+            echo "Error: $cond"
+            exit 1
+        fi
+        if [[ -z "$cond" ]]; then cond=0; fi
+        if [[ "$cond" != 0  &&  "$cond" != 1 ]]; then cond=0; fi
+    fi
+    local stat=$((cond == 0))
+    return $stat
+}
+
+time_period=1
+while getopts "t:h" Option
+do
+  case $Option in
+    t)
+    time_period=$OPTARG
+    ;;
+    h )
+    display_help
+    ;;
+  esac
+done
+
+if float_cond "$time_period == 0"; then
+  echo "Error: time_period has not been set"
+  echo ""
+  display_help
+fi
+
+is_online=`${THOME}/../../tungsten-replicator/bin/trepctl status | grep "state" | grep "ONLINE" | wc -l`
+if float_cond "$is_online == 0"; then
+  echo "CRITICAL: Replicator is not ONLINE"
+  exit $CRITICAL_STATE
+fi
+
+pre_progress_number=`${THOME}/../../tungsten-replicator/bin/trepctl status | grep "appliedLastSeqno" | tr -d "| " | awk -F":" '{print $2}'`
+echo "cluster heartbeat" | ${THOME}/../../tungsten-manager/bin/cctrl > /dev/null
+sleep $time_period
+post_progress_number=`${THOME}/../../tungsten-replicator/bin/trepctl status | grep "appliedLastSeqno" | tr -d "| " | awk -F":" '{print $2}'`
+
+progress_number=`echo "$post_progress_number - $pre_progress_number" | bc -q 2>/dev/null`
+
+if float_cond "$progress_number < 1"; then
+  echo "WARNING: Replicator did not show progress"
+  exit $WARNING_STATE
+fi
+
+echo "OK: Replicator is making progress"
+exit $OK_STATE

data/bin/check_tungsten_services

@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Simple Bash Script To Check Tungsten Services
+# Nagios Plugin For NRPE
+#
+# This script accepts two arguments, {{-r}} and {{-c}}.  The {{-r}} argument 
+# will tell the script to check that the replicator and manager services are 
+# running.  The {{-c}} argument will check the connector service, the 
+# arguments may be given individually or together.  If any service is not 
+# running, the script will return a critical error.
+#
+OK_STATE=0
+WARNING_STATE=1
+CRITICAL_STATE=2
+THOME=`dirname $0`
+
+function display_help
+{
+  echo "Usage: ./check_tungsten_latency [-r] [-c] [-h]"
+  echo " -r     Check the replicator and manager services"
+  echo " -c     Check the connector service"
+  echo " -h     Display this message"
+  exit 0
+}
+
+services_checked=""
+services_checked_glue=""
+services_stopped=""
+services_stopped_glue=""
+stopped_count=0
+
+while getopts "rch" Option
+do
+  case $Option in
+    r	)
+    # Mark that the replicator and manager were checked
+    services_checked="${services_checked}${services_checked_glue}Replicator, Manager"
+    services_checked_glue=", "
+    
+    replicator_running=`${THOME}/../../tungsten-replicator/bin/replicator status | grep "PID" | wc -l`
+    # Check the replicator status
+    if [ $replicator_running -eq 0 ]; then
+  	  services_stopped="${services_stopped}${services_stopped_glue}Replicator"
+  	  services_stopped_glue=", "
+  	  stopped_count=$(($stopped_count+1))
+  	fi
+  	
+    manager_running=`${THOME}/../../tungsten-manager/bin/manager status | grep "PID" | wc -l`
+  	# Check the manager status
+  	if [ $manager_running -eq 0 ]; then
+  	  services_stopped="${services_stopped}${services_stopped_glue}Manager"
+  	  services_stopped_glue=", "
+  	  stopped_count=$(($stopped_count+1))
+  	fi
+  	;;
+  	c )
+  	# Mark that the connector was checked
+    services_checked="${services_checked}${services_checked_glue}Connector"
+    services_checked_glue=", "
+    
+    connector_running=`${THOME}/../../tungsten-connector/bin/connector status | grep "PID" | wc -l`
+    # Check the connector status
+  	if [ $connector_running -eq 0 ]; then
+  	  services_stopped="${services_stopped}${services_stopped_glue}Connector"
+  	  services_stopped_glue=", "
+  	  stopped_count=$(($stopped_count+1))
+  	fi
+    ;;
+    h )
+    display_help
+    ;;
+  esac
+done
+
+# One of the services isn't running
+if [ $stopped_count -gt 0 ]
+then
+  if [ $stopped_count -gt 1 ]; then
+    echo "CRITICAL: $services_stopped are not running"
+  else
+    echo "CRITICAL: $services_stopped is not running"
+  fi
+  
+  exit $CRITICAL_STATE
+fi
+
+# Ensure that something was checked
+if [[ $services_checked == "" ]]; then
+  echo "CRITICAL: No services were checked"
+  exit $CRITICAL_STATE
+fi
+
+# Everything is running
+echo "OK: All services (${services_checked}) are running"
+exit $OK_STATE

data/bin/tungsten_nagios_connector

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+# Copyright (C) 2014 Continuent, Inc.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.  You may obtain
+# a copy of the License at
+# 
+#         http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Initial developer(s): Jeff Mace
+# Contributor(s):
+
+begin
+  require 'rubygems'
+  gem 'continuent-tools-core'
+rescue LoadError
+end
+
+require 'continuent-tools-core'
+require 'continuent-tools-nagios-monitor'
+
+class ContinuentNagiosMonitorConnector
+  include TungstenScript
+  include TungstenNagiosMonitor
+  private
+  
+  def main
+    unless TI.is_running?("connector")
+      critical("The Tungsten Connector is not running")
+    end
+    
+    begin
+      TU.cmd_result("echo '#{opt(:statement)}' | mysql --defaults-file=#{opt(:defaults_file)} -h#{TI.hostname()}")
+    rescue CommandError => ce
+      TU.debug(ce)
+      critical("A connection to the Tungsten Connector could not be created")
+    end
+    
+    ok("The connection was successfully created")
+  end
+  
+  def configure
+    super()
+    
+    add_option(:defaults_file, {
+      :on => "--defaults-file String",
+      :help => "The defaults file to use when connecting to MySQL"
+    })
+    
+    add_option(:statement, {
+      :on => "--statement String",
+      :help => "The command to run against the Tungsten Connector",
+      :default => "tungsten connection status",
+    })
+  end
+  
+  def validate
+    super()
+    
+    unless TU.is_valid?()
+      return TU.is_valid?()
+    end
+    
+    unless TI.is_connector?()
+      unknown("This server is not a Tungsten Connector")
+    end
+    
+    if opt(:defaults_file).to_s() == ""
+      defaults_file = Tempfile.new("tungsten_nagios_connector")
+      opt(:defaults_file, defaults_file.path())
+      
+      defaults_file.puts("[client]")
+      defaults_file.puts("user=#{TI.setting(TI.setting_key(CONNECTORS, "connector_user"))}")
+      defaults_file.puts("password=#{TI.setting(TI.setting_key(CONNECTORS, "connector_password"))}")
+      defaults_file.puts("port=#{TI.setting(TI.setting_key(CONNECTORS, "connector_listen_port"))}")
+      defaults_file.flush()
+    end
+  end
+  
+  def script_name
+    "tungsten_nagios_connector"
+  end
+  
+  self.new().run()
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: continuent-monitors-nagios
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Continuent
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-05 00:00:00.000000000 Z
+date: 2014-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: continuent-tools-monitoring
@@ -27,10 +27,26 @@ dependencies:
 description: 
 email: info@continuent.com
 executables:
+- check_tungsten.sh
+- check_tungsten_backups
+- check_tungsten_latency
+- check_tungsten_online
+- check_tungsten_policy
+- check_tungsten_progress
+- check_tungsten_services
+- tungsten_nagios_connector
 - tungsten_nagios_monitor_threads
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/check_tungsten.sh
+- bin/check_tungsten_backups
+- bin/check_tungsten_latency
+- bin/check_tungsten_online
+- bin/check_tungsten_policy
+- bin/check_tungsten_progress
+- bin/check_tungsten_services
+- bin/tungsten_nagios_connector
 - bin/tungsten_nagios_monitor_threads
 - LICENSE
 - README.md
@@ -54,7 +70,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Continuent Tungsten monitoring scripts for Nagios