rec 1.0.0 → 1.0.1

data/lib/rec/alert.rb

@@ -3,47 +3,53 @@ require 'net/smtp'
 require 'xmpp4r'
 
 module REC
+# Provides the capability to send alerts
+# == mail
+# The simplest approach is to use the native +mail+ program (no credentials required)
+#	Alert.mail(alert)
+#
+# == email and jabber
+# You can also send emails and instant messages via servers, but you'll need to provide
+# credentials to do that.
+# 	Alert.smtp_credentials(user, password, domain, server, port)
+#	Alert.jabber_credentials(user, password, server)
+#
+# Then you can send messages:
+#	Alert.email(alert)
+#	Alert.jabber(alert)
+# or send messages to another recipient, with another subject
+#	Alert.email(alert, you@example.com, "Serious problem")
+#	Alert.jabber(alert, boss@example.com)
+#
+# == Sleeping
+# If you want to avoid being sent instant messages during sleeping hours, you can
+# specify a range of working hours during which urgent alerts may be sent by jabber
+# and outside those working hours the alert will be sent by email instead
+#	Alert.workHours(9,18)		# IMs only between 9am and 6pm
+#	Alert.urgent(alert)		# sent as instant message if during work hours, else by email
+#	Alert.jabber(alert)		# sent as instant message regardless of the time
+#	Alert.normal(alert)		# sent by email
+#
+# == Securing credentials
+# In order to keep email/messaging credentials secure, they can be provided in
+# a file that is only readable by the user executing the rules
+# so the rules script need not contain passwords.
+#	load("/home/rec/alert.conf")
+# Loads the credentials into the rules. The file can contain something like:
+# 	Alert.email_credentials("rec@gmail.com", "tricky", "mydomain.com")
+#	Alert.jabber_credentials("rec@gmail.com", "tricky")
+# <code>/home/rec/alert.conf</code> should be readable only by the otherwise
+# unprivileged user (sec) running the script.
+#
+# While we're on the topic of security, just a reminder that Alert::mail requires no credentials.
 module Alert
 
-	# Provides the capability to send alerts
-	# --mail--
-	# The simplest approach is to use the native +mail+ program (no credentials required)
-	# Alert.mail(alert)
-	#
-	# --email and jabber--
-	# You can also send emails and instant messages via servers, but you'll need to provide
-	# credentials to do that.
-	# - Alert.smtp_credentials(user, password, domain, server, port)
-	# - Alert.jabber_credentials(user, password, server)
-	#
-	# Then you can send messages:
-	# - Alert.email(alert)
-	# - Alert.jabber(alert)
-	# or send messages to another recipient, with another subject
-	# - Alert.email(alert, you@example.com, "Serious problem")
-	# - Alert.jabber(alert, boss@example.com)
-	#
-	# --Sleeping--
-	# If you want to avoid being sent instant messages during sleeping hours, you can
-	# specify a range of working hours during which urgent alerts may be sent by jabber
-	# and outside those working hours the alert will be sent by email instead
-	# Alert.workHours(9,18)		# IMs only between 9am and 6pm
-	# Alert.urgent(alert)		# sent as instant message if during work hours, else by email
-	# Alert.jabber(alert)		# sent as instant message regardless of the time
-	# Alert.normal(alert)		# sent by email
-
-
+	# Sets the default subject for alerts, overriding the system default of "Alert"
 	def self.default_subject(subject)
 		@@defaultSubject = subject
 	end
 	@@defaultSubject = "Alert"
 
-	#load("/home/rec/alert.conf") can contain something like this:
-	#  Alert.email_credentials("rec@gmail.com", "tricky", "mydomain.com")
-	#  Alert.jabber_credentials("rec@gmail.com", "tricky")
-	#so the rules script need not contain passwords.
-	#/home/rec/alert.conf should be readable only by the otherwise unprivileged user (sec)
-	# running the script
 
 	# provides the credentials needed for sending email
 	def self.smtp_credentials(user, password, domain, server="smtp.gmail.com", port=587)
@@ -61,10 +67,12 @@ module Alert
 		@@jabberServer = server
 	end
 
+	# sets the email address to receive alerts
 	def self.emailTo=(address)
 		@@emailTo = address
 	end
 
+	# sets the jabber address to receive alerts
 	def self.jabberTo=(address)
 		@@jabberTo = address
 	end
@@ -92,7 +100,7 @@ module Alert
 	end
 
 	# define the working hours during which instant messages are allowed
-	# Note that Alert.work_hours(7,21) means "7am-9pm" as you would think, so from 07:00 to 20:59
+	# Note that Alert.work_hours(7,21) means "7am-9pm" as you would assume, so from 07:00 to 20:59
 	def self.work_hours(start, finish)
 		@@workHours = start..finish
 	end
@@ -107,6 +115,7 @@ module Alert
 		end
 	end
 
+	# Alias for Alert::email
 	def self.normal(alert)
 		self.email(alert)
 	end

data/lib/rec/correlator.rb

@@ -1,22 +1,33 @@
 require 'rec/rule'
 
 module REC
+
+# The Correlator reads in log entries, matching them against the ruleset,
+# creates states as necessary, generates new (correlated) events, and sends alerts.
 class Correlator
 
 	@@eventsIn = 0
 	@@eventsMissed = 0
 
+	# Convenience method to create and start a correlator. Possible options are:
+	# - :debug => true
 	def self.start(opts={})
 		$debug = opts[:debug] || false
 		self.new().start()
 	end
 
+	# Create a new Correlator
 	def initialize()
 		@time = @startupTime = Time.now()
 		@year = @startupTime.year
 		@running = false
 	end
 
+	# Start a Correlator. +INT+ and +TERM+ signals will stop the Correlator.
+	# +USR1+ signal will cause Correlator to display statistics and continue running.
+	#
+	# Missed events are written to IO Stream 3, in case they are of interest
+	# (typically while testing rulesets)
 	def start()
 		Signal.trap("INT")  { finish() }
 		Signal.trap("TERM") { finish() }
@@ -24,7 +35,7 @@ class Correlator
 			stats()
 			run()
 		}
-		$stderr.puts("srec is starting...")
+		$stderr.puts("rec is starting...")
 		begin
 			$miss = IO.open(3, "a")		# for missed events
 		rescue
@@ -34,6 +45,9 @@ class Correlator
 		run()
 	end
 
+	# reads the next input log entry, parses it into a timestamp and a message,
+	# and checks the message against the ruleset. Continuously loops until
+	# the log input stream is closed, or interrupted by a signal
 	def run()
 		while @running and !$stdin.eof? do
 			logLine = gets()
@@ -63,15 +77,21 @@ class Correlator
 		finish() if $stdin.eof?
 	end
 
+	# Stop correlating, close IO streams and exit.
 	def finish()
 		@running = false
 		$miss.close() unless $miss.nil? or $miss.closed?
 		# NOTE: some states may have something useful to say, or maybe we could store them
 		stats()
-		$stderr.puts("srec is finished.")
+		$stderr.puts("rec is finished.")
 		exit 0
 	end
 
+	# Parses a log entry into a timestamp and a message. Handles formats like:
+	#	Apr 22 16:40:18 aqua Firewall[205]: ...
+	#	[err] Fri Dec 30 23:58:56 2011 - scan error: ...
+	#	2012-04-22 08:43:22.099 EST - Module: ...
+	# otherwise time stands still.
 	def parse(logLine)
 		if logLine =~ /^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d+)\s(\d\d)\:(\d\d)\:(\d\d)/
 			# Apr 22 16:40:18 aqua Firewall[205]: Skype is listening from 0.0.0.0:51304 proto=6
@@ -96,8 +116,8 @@ class Correlator
 		[time, message]
 	end
 
+	# Reports statistics to stderr
 	def stats()
-		# report statistics to stderr
 		checked, matched, created, reacted, rules = Rule.stats()
 		statesCount, eventsOut = State.stats()
 		$stderr.puts("-"*40)

data/lib/rec/mock-alert.rb

@@ -1,4 +1,5 @@
 module REC
+# :nodoc:all
 # mock the Alert class for testing purposes
 class Alert
 

data/lib/rec/rule.rb

@@ -1,19 +1,24 @@
 require 'rec/state'
 
 module REC
+
+# A Rule specifies which log entries to match, what to remember, and what to do about them.
 class Rule
 
 	@@rules = []
-	@@index = {}	# index of rules to allow lookup of messages etc.
-	@@checked = {}	# number of times each rule was checked
-	@@matched = {}	# number of times each rule was matched
-	@@created = {}	# number of states created by each rule
-	@@reacted = {}	# number of times #react was called on each rule
+	# hash index of rules to allow lookup of messages etc.
+	@@index = {}
+	@@checked = {}
+	@@matched = {}
+	@@created = {}
+	@@reacted = {}
 
+	# Convenience method to iterate through the ruleset
 	def self.each(&block)
 		@@rules.each(&block)
 	end
 
+	# Adds a rule to the ruleset
 	def self.<<(rule)
 		@@rules << rule
 		@@index[rule.rid] = rule
@@ -23,16 +28,52 @@ class Rule
 		@@reacted[rule.rid] = 0
 	end
 
+	# Get a rule belonging to the key of +rid+
 	def self.[](rid)
 		@@index[rid]
 	end
 
+	# Returns some summary statistics in a 5-element array, the first four elements
+	# a hash keyed on rule ID, the fifth is an array of rules:
+	# 1. number of times each rule was checked
+	# 2. number of times each rule was matched
+	# 3. number of states created by each rule
+	# 4. number of times #react was called on each rule
+	# 5. list of rules, evaluated in sequence for each event
 	def self.stats()
 		[@@checked, @@matched, @@created, @@reacted, @@rules]
 	end
 
-	attr_reader(:rid, :pattern, :message, :lifespan, :alert, :params, :action)
+	# the unique ID of the rule
+	attr_reader :rid
+	# the regexp pattern to match an original log entry against
+	#	:pattern => /^\s\w+\sFirewall\[\d+\]\:\sSkype is listening from 0.0.0.0:(\d+)/,
+	#	:details => ["port"],
+	# Note that regexp captures must correspond to customer field names in +details+
+	attr_reader :pattern
+	# the template for the title of any state created.
+	#		:message => "sudo activity for user %userid$s",
+	# will create states with titles like "sudo activity for user richard"
+	attr_reader :message
+	# the time in seconds for a created state to persist
+	attr_reader :lifespan
+	# the template for an alert message to be generated should it be necessary
+	attr_reader :alert
+	# hash of the rules parameters, passed in when creating the rule
+	attr_reader :params
+	# block to be executed when #react is called. For example:
+	#	Rule.new(10035, {
+	#		:pattern => /^\s\w+\sFirewall\[\d+\]\:\sSkype is listening from 0.0.0.0:(\d+)/,
+	#		:details => ["port"],
+	#		:message => "Skype conversation started on port %port$d",
+	#		:alert => "Skype running on port %port$d",
+	#		:lifespan => 479
+	#	}) { |state|
+	#		state.alert_first_only()
+	#	}
+	attr_reader :action
 
+	# Creates a new rule. +rid+ must be unique.
 	def initialize(rid, params={}, &action)
 		@rid = rid
 		@pattern = params[:pattern] || raise("No pattern specified for rule #{@ruleId}")
@@ -49,6 +90,7 @@ class Rule
 		Rule << self
 	end
 
+	# Checks the original +logMessage+ against the rule, looking for a match.
 	def check(logMessage)
 		@@checked[@rid] += 1
 		matchData = @pattern.match(logMessage) || return
@@ -64,12 +106,14 @@ class Rule
 		return(title)
 	end
 
+	# Creates a state with the given title at the specified time
 	def create_state(title, time)
 		@@created[@rid] += 1
 		$stderr.puts("+ Creating new state  #{title}") if $debug
 		State.new(title, time, @lifespan, @params)
 	end
 
+	# Executes any action specified by the rule
 	def react(state, time, logLine)
 		@@reacted[@rid] += 1
 		state.update(time, @rid, @matches, @alert, logLine)
@@ -77,6 +121,7 @@ class Rule
 		@action.call(state) if @action
 	end
 
+	# Returns the +continue+ parameter. If false, stop processing rules for this event.
 	def continue()
 		@params[:continue]
 	end

data/lib/rec/state.rb

@@ -1,18 +1,42 @@
 require 'time'
 
 module REC
+
+# A State is an object that represents the memory of something having happened.
+# For example, "server terra is down".
+# It also remembers useful statistics about what caused this state to be (the
+# original log entries and the rule they matched),
+# for how long it should remain in memory, what it pertains to (eg. the server
+# called 'terra').
+#
+# A state is also useful for other rules to refer to. For example, a second rule
+# matching "host terra is up" can check if the server is currently down by reference
+# to the state with a title of "host terra is down".
+#
+# This is much more useful than matching log entries one by one without any memory
+# of what has gone before. You cannot *correlate* events without keeping State.
 class State
 
+	# A array of Timeouts. A Timeout struct has two elements:
+	# - timestamp at which to expire
+	# - key of the state to be expired
 	@@timeouts = []
+
+	# A hash of states, keyed on state title
 	@@states = {}
+
+	# A count of new events sent to output
 	@@eventsOut = 0
 
 	Struct.new("Timeout", :expiry, :key)
 
+	# Returns the state matching the given key (title)
 	def self.[](key)
 		@@states[key]
 	end
 
+	# Add a Timeout for the given time and specified state. This timeout is sorted
+	# into the correct sequence of timeouts to make State::expire_states more efficient.
 	def self.timeout_at(time, title)
 		tnew = Struct::Timeout.new(time, title)
 		n = @@timeouts.find_index { |to|
@@ -25,6 +49,7 @@ class State
 		end
 	end
 
+	# Deletes all expired states
 	def self.expire_states(time)
 		timeout = @@timeouts.first()
 		while @@timeouts.length > 0 and timeout.expiry < time do
@@ -40,13 +65,41 @@ class State
 		end
 	end
 
+	# Returns a 2-element array containing:
+	# - the number of states
+	# - the number of new events sent to output
 	def self.stats()
 		statesCount = @@states.keys.length
 		[statesCount, @@eventsOut]
 	end
 
-	attr_reader(:rid, :title, :lifespan, :alert, :params, :count, :created, :updated, :dur, :details)
+	# unique ID of the rule which gave rise to this state
+	attr_reader :rid
+	# The unique title for this state (eg. "server earth is down")
+	attr_reader :title
+	# How long this state shoudl live before being automatically forgotten
+	attr_reader :lifespan
+	# An alert message to be sent if subsequent events warrant it
+	attr_reader :alert
+	# Hash of parameters of the rule that created this state
+	attr_reader :params
+	# Number of times this state has been matched
+	attr_reader :count
+	# time when this state was created
+	attr_reader :created
+	# last time this state was updated
+	attr_reader :updated
+	# age of this state
+	attr_reader :dur
+	# Hash of custom fields to be remembered
+	attr_reader :details
 
+	# Creates a new state with the given (unique) +title+, the +time+ of creation,
+	# the +lifespan+ before the state is forgotten, and a hash of parameters.
+	#
+	# Note that the time is not necessarily 'now' because REC can be executed against
+	# historical log files. It uses the timestamp of the original log entry, not the
+	# current clock time.
 	def initialize(title, time, lifespan, params={})
 		@title = title
 		@lifespan = lifespan.to_f
@@ -63,6 +116,15 @@ class State
 		State.timeout_at(time + @lifespan, @title)
 	end
 
+	# Updates the statistics of this state (following a match).
+	# Remembers lots of useful things:
+	# -	the number of matches so far
+	# - time last updated
+	# - age of this state
+	# - the ID of the rule which last matched
+	# - more details (custom fields) may be added to memory
+	# - the alert message
+	# - a list of the original log entries pertaining to this state
 	def update(time, rid, matches, alert, logLine=nil)
 		@count = @count.succ
 		@updated = time
@@ -73,14 +135,33 @@ class State
 		@logs << logLine if @params[:capture]
 	end
 
-	def release()
-		@@states.delete(@title)
+	# Forget a state.
+	# -	if no pattern is provided, forget this state.
+	# -	if a pattern is provided, use the stats for this state to determine the title
+	#	of the other state, and remove that from memory.
+	# For example, if the server is back up again, we no longer need to remember that
+	# the server was down.
+	def release(pattern=nil)
+		if pattern.nil?
+			@@states.delete(@title)
+		else
+			@@states.delete(pattern.sprinth(stats))
+		end
 	end
 
+	# Returns the details of the state (ie. whatever custom fields were defined, eg. userid)
+	# merged with the standard statistics:
+	# - count: number of matches so far
+	# - dur: age of the state
+	# - created: time when this state was created
+	# - updated: time last updated (eg. when the latest event matched this state)
 	def stats()
 		@details.merge({"count"=>@count, "dur"=>@dur, "created"=>@created, "updated"=>@updated})
 	end
 
+	# Creates a new event, writes it to the output log, and returns the event.
+	#
+	# An *event* (or 'log entry') is a timestamp followed by a *message*
 	def generate_alert()
 		message = @alert.sprinth(stats())
 		event = "%s %s" % [@created.iso8601, message] + @logs.join("\n")
@@ -89,11 +170,16 @@ class State
 		@@eventsOut = @@eventsOut + 1
 		event
 	end
-	
+
+	# Creates a new event when this state is created, but ignores later occurrences
 	def alert_first_only()
 		generate_alert() if @count == 1
 	end
 
+	# Allow access to any parameter by a convenience method
+	# 	state.capture
+	# is more succinct than
+	#	state.params['capture']
 	def method_missing(symbol, *args)
 		@params[symbol]
 	end

data/lib/rec.rb

@@ -2,3 +2,131 @@
 require 'string'
 require 'rec/correlator'
 require 'rec/alert'
+
+=begin rdoc
+= Ruby Event Correlation
+REC correlates events in order to generate a smaller set of more meaningful events.
+
+== Installation
+1. Install the gem
+	$ sudo gem install rec
+
+2. Select a ruleset or create your own
+	#!/usr/bin/ruby
+	require 'rec'
+	include REC
+	require 'rulesets/postfix-rules'
+	Correlator::start()
+
+3. Start it up
+	$ rulesets/rules.rb < /var/log/mail.log 3>missed.log 2>control.log > newevents.log
+
+== Why correlate events?
+We all know that we should read our log files. But reading log files is *really* boring,
+and frankly its easy to miss important things in all the superfluous detail.
+
+[Save time]
+	If you are lazy enough to not want to review all of your log files manually forever, and
+	smart enough to work out what needs monitoring and when you might want to pay attention,
+	then wouldn't it be good if you could define those rules and let the computer do what it
+	does best?
+
+[Generate meaning]
+	The logs of many applications are filled with entries that are quite low level - perhaps
+	wonderful for debugging, but typically not terribly meaningful in terms of business.
+	Wouldn't it be good if we could summarise a bunch of low level events into a single
+	business event - and then just read the <em>business log</em>.
+
+== Alternatives
+There are several alternatives to REC which may suit your needs better:
+* splunk[www.splunk.com]
+* nagios[www.nagios.com]
+* scalextreme.com[www.scalextreme.com]
+While I like these options, I find they take a lot of configuring.
+They also has some dependencies that make them a bit heavier than you may want.
+If you just want to keep track of a few kinds of events, want a lot of flexibility
+and control without too much effort, then REC may be of some value.
+
+== How does REC work?
+Each entry in a log file is an *event*.
+The Correlator reads the events, and attempts to match an event against each Rule.
+If an event matches a rule, the rule creates a State which just means we're remembering
+that the event matched a rule. The pattern to match is a regexp, and the captured values
+are named. For example
+	# log entry => "nfs: server earth not responding"
+	pattern => /nfs\: server (\w+) not responding/
+	details => ['host']
+	# values of interest are captured into a hash => {'host' => 'earth' }
+	:message => "Server %host$s is down"
+	# interpolation with named parameters => "Server earth is down"
+
+A state has a fixed lifetime, set when it is created. At the end of its life, it may simply
+expire quietly, or a pre-defined action may be executed. For example, if we find a server is down,
+we may want to wait for 3 minutes and if it is not up again, then alert the administrator.
+The server being down is a state, and two states are distinguished by their *titles*. For example,
+"host earth is down" and "host terra is down".
+
+Now that we're remembering a set of states, we can match events against not only the event's
+message, but also other states. For example, we can match "host terra is up" against a previously
+created state "host terra is down", and generate a new event "host terra is back up after 14 minutes".
+We can also 'swallow' all of the rest of the "host terra is down" events because they add nothing new.
+This <em>event compression</em> means the administrator gets one important message, and not 27
+distracting alerts.
+
+An Alert can be sent by email or IM, depending on your preferences and working hours.
+The destinations and credentials are supplied to your ruleset:
+	# For better security, move the next few lines into a file readable only by
+	# the user running this script eg. /home/rec/alert.conf
+	# and then require that file
+	Alert.smtp_credentials("rec@gmail.com", "recret", "myfirm.com")
+	Alert.emailTo = "me@myfirm.com"
+	Alert.jabber_credentials("rec@gmail.com", "recret")
+	Alert.jabberTo = "me@myfirm.com"
+
+Rules can then send an alert when desired. Two common cases involve alerting immediately
+on the first event (eg. "host terra is down"), and alerting on expiry or at a subsequent event
+(eg. "host terra is back up").
+	state.alert_first_only()		# => generate a new event on first original event
+						# or
+	Alert.normal(state.alert_first_only())	# => log and also send the new event via email
+
+In most cases, however, it is not necessary to alert the administrator at all. It is enough to
+log the new event in the output logfile for later review.
+
+== Examples
+The best way to understand REC is to see how rules are written. The following examples were
+inspired by SEC (simple-evcorr.sourceforge.net), so they employ the same names.
+
+1.	Warn if an user is having trouble executing sudo commands
+	The log entry (/var/log/secure) looks like this:
+
+		Sep 16 07:09:22 earth sudo:  richard : 3 incorrect password attempts ;...
+
+	and the rule might look like this:
+
+		# single threshold rule
+		Rule.new(10034, {
+			:pattern => /\w+ sudo\:  (\w+) \: 3 incorrect password attempts/,
+			:details => ["userid"],
+			:message => "Failed sudo password for user %userid$s",
+			:lifespan => 60,
+			:alert => "'Too much sudo activity' userid=%userid$s attempts=%count$d dur=%dur$0.3fs ",
+			:threshold => 3,
+			:capture => true
+		}) { |state|
+		if state.count == state.threshold
+			Alert.urgent(state.generate_alert())
+			state.release()
+		end
+		}
+
+	This rule matches the log entry against the *pattern*. Note that the captured value
+	<code>(\w+)</code> is stored as the +userid+. The +details+ parameter specifies the names
+	of the captured values in the sequence they appear in the pattern.
+
+	The *message* specifies the title of the state that is created. If there is no title
+
+
+=end
+module REC
+end

data/lib/string.rb

@@ -1,9 +1,11 @@
+# Custom extension to String
 class String
-	# interpolate hash values into a formatted string
-	# The string should have a form like "Stats uid %uid$-5d belongs to %userid$s"
-	# '%uid$-5d' is replaced by '%-5d' and '%userid$s' becomes '%s'
-	# and the relevant values are pulled from the hash into an array
-	# and interpolated using normal sprintf features
+	# Interpolates hash values into a formatted string.
+	#	s = "Stats uid %uid$-5d belongs to %userid$s"
+	#	h = {'uid': 501, 'userid': 'richard}
+	#	s.sprinth(h)
+	#	# intermediate => "Stats uid %-5d belongs to %s" % [501, 'richard']
+	#	# finally      => "Stats uid   501 belongs to richard"
 	def sprinth(hash={})
 		raise ArgumentError.new("sprinth argument must be a Hash") unless hash.is_a?(Hash)
 		self.gsub(/\%\w+\$/,"%") % self.scan(/\%\w+\$/).collect { |token| hash[token[1..-2]] }

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 1
   - 0
-  - 0
-  version: 1.0.0
+  - 1
+  version: 1.0.1
 platform: ruby
 authors: 
 - Richard Kernahan
@@ -25,7 +25,7 @@ description: "\tSifts through your log files in real time, using stateful intell
   \tbut is original code and any defects are entirely mine.\n\
   \tWhile event correlation is inherently complex, REC attempts to make common tasks easy\n\
   \twhile preserving plenty of power and flexibility for ambitious tasks.\n"
-email: rec@finalstep.com.au
+email: dev.rec@finalstep.com.au
 executables: []
 
 extensions: []