rec 1.0.0

data/lib/rec/alert.rb

@@ -0,0 +1,115 @@
+require 'rubygems'
+require 'net/smtp'
+require 'xmpp4r'
+
+module REC
+module Alert
+
+	# Provides the capability to send alerts
+	# --mail--
+	# The simplest approach is to use the native +mail+ program (no credentials required)
+	# Alert.mail(alert)
+	#
+	# --email and jabber--
+	# You can also send emails and instant messages via servers, but you'll need to provide
+	# credentials to do that.
+	# - Alert.smtp_credentials(user, password, domain, server, port)
+	# - Alert.jabber_credentials(user, password, server)
+	#
+	# Then you can send messages:
+	# - Alert.email(alert)
+	# - Alert.jabber(alert)
+	# or send messages to another recipient, with another subject
+	# - Alert.email(alert, you@example.com, "Serious problem")
+	# - Alert.jabber(alert, boss@example.com)
+	#
+	# --Sleeping--
+	# If you want to avoid being sent instant messages during sleeping hours, you can
+	# specify a range of working hours during which urgent alerts may be sent by jabber
+	# and outside those working hours the alert will be sent by email instead
+	# Alert.workHours(9,18)		# IMs only between 9am and 6pm
+	# Alert.urgent(alert)		# sent as instant message if during work hours, else by email
+	# Alert.jabber(alert)		# sent as instant message regardless of the time
+	# Alert.normal(alert)		# sent by email
+
+
+	def self.default_subject(subject)
+		@@defaultSubject = subject
+	end
+	@@defaultSubject = "Alert"
+
+	#load("/home/rec/alert.conf") can contain something like this:
+	#  Alert.email_credentials("rec@gmail.com", "tricky", "mydomain.com")
+	#  Alert.jabber_credentials("rec@gmail.com", "tricky")
+	#so the rules script need not contain passwords.
+	#/home/rec/alert.conf should be readable only by the otherwise unprivileged user (sec)
+	# running the script
+
+	# provides the credentials needed for sending email
+	def self.smtp_credentials(user, password, domain, server="smtp.gmail.com", port=587)
+		@@emailUser = user
+		@@emailPassword = password
+		@@emailDomain = domain
+		@@emailServer = server
+		@@emailPort = port
+	end
+
+	# provides the credentials needed for sending instant messages
+	def self.jabber_credentials(user, password, server="talk.google.com")
+		@@jabberUser = user
+		@@jabberPassword = password
+		@@jabberServer = server
+	end
+
+	def self.emailTo=(address)
+		@@emailTo = address
+	end
+
+	def self.jabberTo=(address)
+		@@jabberTo = address
+	end
+
+	# Send the alert via local mailer
+	def self.mail(alert, recipient=@@emailTo, subject=@@defaultSubject)
+		`echo "#{alert}" | mail -s #{subject.gsub(/\s/,'\ ')} #{recipient} 1<&2`
+	end
+
+	# Send the alert via an email server
+	def self.email(alert, recipient=@@emailTo, subject=@@defaultSubject)
+		smtp = Net::SMTP.new(@@emailServer, @@emailPort)
+		smtp.enable_starttls()
+		smtp.start(@@emailDomain, @@emailUsername, @@emailPassword, :plain)
+		smtp.send_message(alert, @@emailUsername, recipient)
+	end
+
+	# Send the alert via google talk (or any other XMPP service)
+	def self.jabber(alert, recipient=@@jabberTo, subject=@@defaultSubject)
+		client = Jabber::Client::new(Jabber::JID.new(@@JabberUser))
+		client.connect(@@jabberServer)
+		client.auth(@@jabberPassword)
+		message = Jabber::Message::new(recipient, alert).set_type(:normal).set_id('1').set_subject(subject)
+		client.send(message)
+	end
+
+	# define the working hours during which instant messages are allowed
+	# Note that Alert.work_hours(7,21) means "7am-9pm" as you would think, so from 07:00 to 20:59
+	def self.work_hours(start, finish)
+		@@workHours = start..finish
+	end
+	@@workHours = 7...21
+
+	# Send an instant message during work hours, else send an email
+	def self.urgent(alert)
+		if @@workhours.include?(Time.now.hour)
+			self.jabber(alert)
+		else
+			self.email(alert)
+		end
+	end
+
+	def self.normal(alert)
+		self.email(alert)
+	end
+
+end
+end

data/lib/rec/correlator.rb

@@ -0,0 +1,121 @@
+require 'rec/rule'
+
+module REC
+class Correlator
+
+	@@eventsIn = 0
+	@@eventsMissed = 0
+
+	def self.start(opts={})
+		$debug = opts[:debug] || false
+		self.new().start()
+	end
+
+	def initialize()
+		@time = @startupTime = Time.now()
+		@year = @startupTime.year
+		@running = false
+	end
+
+	def start()
+		Signal.trap("INT")  { finish() }
+		Signal.trap("TERM") { finish() }
+		Signal.trap("USR1") {
+			stats()
+			run()
+		}
+		$stderr.puts("srec is starting...")
+		begin
+			$miss = IO.open(3, "a")		# for missed events
+		rescue
+			$miss = nil
+		end
+		@running = true
+		run()
+	end
+
+	def run()
+		while @running and !$stdin.eof? do
+			logLine = gets()
+			next if logLine.nil?
+			logLine.strip!()
+			next if logLine.empty?
+			@@eventsIn += 1
+			@time, message = parse(logLine)
+			$stderr.puts("< "+message) if $debug
+			State.expire_states(@time)	# remove expired states before we check the rules
+			eventMatched = false
+			Rule.each { |rule|
+				title = rule.check(message)
+				eventMatched = true unless title.nil?	# empty match is still a match
+				next if title.nil?
+				break if title.empty?	# match without a message means 'swallow this event'
+				state = State[title] || rule.create_state(title, @time)
+				rule.react(state, @time, logLine)
+				$stderr.puts("breaking after rule #{rule.rid}") unless (!$debug or rule.continue())
+				break unless rule.continue()
+			}
+			if !eventMatched
+				@@eventsMissed += 1
+				$miss.puts("* "+logLine) unless $miss.nil?
+			end
+		end
+		finish() if $stdin.eof?
+	end
+
+	def finish()
+		@running = false
+		$miss.close() unless $miss.nil? or $miss.closed?
+		# NOTE: some states may have something useful to say, or maybe we could store them
+		stats()
+		$stderr.puts("srec is finished.")
+		exit 0
+	end
+
+	def parse(logLine)
+		if logLine =~ /^(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d+)\s(\d\d)\:(\d\d)\:(\d\d)/
+			# Apr 22 16:40:18 aqua Firewall[205]: Skype is listening from 0.0.0.0:51304 proto=6
+			time = Time.local(@year, $1, $2, $3, $4, $5, 0)
+			message = $'
+		elsif logLine =~ /^\[\w+\]\s[Sun|Mon|Tue|Wed|Thu|Fri|Sat]\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d+)\s(\d\d)\:(\d\d)\:(\d\d)\s(\d{4})/
+			# [err] Fri Dec 30 23:58:56 2011 - scan error: 451 SCAN Engine error 2 ...
+			time = Time.local($6, $1, $2, $3, $4, $5, 0)
+			message = $'
+		elseif logLine =~ /^(\d{4})\-(\d\d)\-(\d\d)\s(\d\d)\:(\d\d)\:(\d\d)\.(\d+)\s(\w+)/
+			# 2012-04-22 08:43:22.099 EST - Module: PlistFile ...
+			if $7 == "UTC" or $7 == "GMT"
+				time = Time.utc($1, $2, $3, $4, $5, $6, $7)
+			else
+				time = Time.local($1, $2, $3, $4, $5, $6, $7)
+			end
+			message = $'
+		else
+			time = @time	# time stands still
+			message = logLine
+		end
+		[time, message]
+	end
+
+	def stats()
+		# report statistics to stderr
+		checked, matched, created, reacted, rules = Rule.stats()
+		statesCount, eventsOut = State.stats()
+		$stderr.puts("-"*40)
+		$stderr.puts("srec has been running for %.1fs" % [Time.now - @startupTime])
+		$stderr.puts("events:   in      %-8d out     %-8d missed  %-8d" % [@@eventsIn, eventsOut, @@eventsMissed])
+		$stderr.puts("states:   active  %-8d created %-8d " % [statesCount, created.values.reduce(:+)])
+		$stderr.puts("rules:    checked %-8d matched %-8d reacted %-8d" % [checked.values.reduce(:+),matched.values.reduce(:+), reacted.values.reduce(:+)])
+		$stderr.puts("Rule ID  checked  matched  freq %   reacted ")
+		#checked.keys.sort { |a,b| matched[b] <=> matched[a]	}.each {|rid| # descending by matches
+		rules.collect { |rule| rule.rid }.each { |rid|
+			if checked[rid] > 0
+				freq = "%5.2f" % [matched[rid].to_f() / checked[rid].to_f() * 100]
+			else
+				freq = "never"
+			end
+			$stderr.puts("%-8d %-8d %-8d %-8s %-8d" % [rid, checked[rid], matched[rid], freq, reacted[rid]])
+		}
+	end
+
+end
+end

data/lib/rec/mock-alert.rb

@@ -0,0 +1,25 @@
+module REC
+# mock the Alert class for testing purposes
+class Alert
+
+	@@emailsSent = []
+	@@jabbersSent = []
+
+	def self.email(alert, recipient=@@emailTo, subject=@@defaultSubject)
+		@@emailsSent << [alert, recipient, subject]
+	end
+
+	def self.jabber(alert, recipient=@@jabberTo, subject=@@defaultSubject)
+		@@jabbersSent << [alert, recipient, subject]
+	end
+
+	def emailsSent()
+		@@emailsSent
+	end
+
+	def jabbersSent()
+		@@jabberSent
+	end
+
+end
+end

data/lib/rec/rule.rb

@@ -0,0 +1,85 @@
+require 'rec/state'
+
+module REC
+class Rule
+
+	@@rules = []
+	@@index = {}	# index of rules to allow lookup of messages etc.
+	@@checked = {}	# number of times each rule was checked
+	@@matched = {}	# number of times each rule was matched
+	@@created = {}	# number of states created by each rule
+	@@reacted = {}	# number of times #react was called on each rule
+
+	def self.each(&block)
+		@@rules.each(&block)
+	end
+
+	def self.<<(rule)
+		@@rules << rule
+		@@index[rule.rid] = rule
+		@@checked[rule.rid] = 0
+		@@matched[rule.rid] = 0
+		@@created[rule.rid] = 0
+		@@reacted[rule.rid] = 0
+	end
+
+	def self.[](rid)
+		@@index[rid]
+	end
+
+	def self.stats()
+		[@@checked, @@matched, @@created, @@reacted, @@rules]
+	end
+
+	attr_reader(:rid, :pattern, :message, :lifespan, :alert, :params, :action)
+
+	def initialize(rid, params={}, &action)
+		@rid = rid
+		@pattern = params[:pattern] || raise("No pattern specified for rule #{@ruleId}")
+		@message = params[:message] || ""	# no message means no state created - ie. ignore event
+		@lifespan = params[:lifespan] || 0
+		@alert = params[:alert] || @message
+		@allstates = params[:allstates] || []
+		@anystates = params[:anystates] || []
+		@notstates = params[:notstates] || []
+		@details = params[:details] || []
+		@params = params
+		@action = action
+		@matches = nil
+		Rule << self
+	end
+
+	def check(logMessage)
+		@@checked[@rid] += 1
+		matchData = @pattern.match(logMessage) || return
+		@matches = Hash[@details.zip(matchData.to_a()[1..-1])]	# map matched values to detail keys
+		# if any notstates are specified, make sure none are present
+		@notstates.each { |str| return if State[str.sprinth(@matches)]	}
+		# if any allstates are specified, they all need to be present
+		@allstates.each { |str| return unless State[str.sprinth(@matches)]	}
+		# if anystates are specified, we must find one that matches
+		return unless @anystates.empty? or @anystates.detect {|str| State[str.sprinth(@matches)] }
+		title = @message.sprinth(@matches)
+		@@matched[@rid] += 1
+		return(title)
+	end
+
+	def create_state(title, time)
+		@@created[@rid] += 1
+		$stderr.puts("+ Creating new state  #{title}") if $debug
+		State.new(title, time, @lifespan, @params)
+	end
+
+	def react(state, time, logLine)
+		@@reacted[@rid] += 1
+		state.update(time, @rid, @matches, @alert, logLine)
+		$stderr.puts("~ Rule #{@rid}, state = #{state.inspect()}") if $debug
+		@action.call(state) if @action
+	end
+
+	def continue()
+		@params[:continue]
+	end
+
+end
+end

data/lib/rec/state.rb

@@ -0,0 +1,102 @@
+require 'time'
+
+module REC
+class State
+
+	@@timeouts = []
+	@@states = {}
+	@@eventsOut = 0
+
+	Struct.new("Timeout", :expiry, :key)
+
+	def self.[](key)
+		@@states[key]
+	end
+
+	def self.timeout_at(time, title)
+		tnew = Struct::Timeout.new(time, title)
+		n = @@timeouts.find_index { |to|
+			to.expiry > time
+		}
+		if n.nil?
+			@@timeouts = @@timeouts << tnew
+		else
+			@@timeouts[n..n] = [tnew, @@timeouts[n]]
+		end
+	end
+
+	def self.expire_states(time)
+		timeout = @@timeouts.first()
+		while @@timeouts.length > 0 and timeout.expiry < time do
+			state = State[timeout.key]
+			if state.nil?
+				@@timeouts.shift
+				timeout = @@timeouts.first()
+				next
+			end
+			#$stderr.puts("Releasing state #{state.title} with count of #{state.count}")
+			@@states.delete(@@timeouts.shift().key)
+			timeout = @@timeouts.first()
+		end
+	end
+
+	def self.stats()
+		statesCount = @@states.keys.length
+		[statesCount, @@eventsOut]
+	end
+
+	attr_reader(:rid, :title, :lifespan, :alert, :params, :count, :created, :updated, :dur, :details)
+
+	def initialize(title, time, lifespan, params={})
+		@title = title
+		@lifespan = lifespan.to_f
+		@params = params
+		@count = 0
+		@created = time
+		@updated = time
+		@dur = 0
+		@rid = 0
+		@alert = ""
+		@logs = []		# array of remembered logLines
+		@details = {}	# hash of remembered details
+		@@states[title] = self
+		State.timeout_at(time + @lifespan, @title)
+	end
+
+	def update(time, rid, matches, alert, logLine=nil)
+		@count = @count.succ
+		@updated = time
+		@dur = @updated - @created
+		@rid = rid
+		@details.merge!(matches)
+		@alert = alert
+		@logs << logLine if @params[:capture]
+	end
+
+	def release()
+		@@states.delete(@title)
+	end
+
+	def stats()
+		@details.merge({"count"=>@count, "dur"=>@dur, "created"=>@created, "updated"=>@updated})
+	end
+
+	def generate_alert()
+		message = @alert.sprinth(stats())
+		event = "%s %s" % [@created.iso8601, message] + @logs.join("\n")
+		print("> ") if $debug
+		puts(event)
+		@@eventsOut = @@eventsOut + 1
+		event
+	end
+	
+	def alert_first_only()
+		generate_alert() if @count == 1
+	end
+
+	def method_missing(symbol, *args)
+		@params[symbol]
+	end
+
+end
+end

data/lib/rec.rb

@@ -0,0 +1,4 @@
+# Ruby Event Correlation
+require 'string'
+require 'rec/correlator'
+require 'rec/alert'

data/lib/string.rb

@@ -0,0 +1,11 @@
+class String
+	# interpolate hash values into a formatted string
+	# The string should have a form like "Stats uid %uid$-5d belongs to %userid$s"
+	# '%uid$-5d' is replaced by '%-5d' and '%userid$s' becomes '%s'
+	# and the relevant values are pulled from the hash into an array
+	# and interpolated using normal sprintf features
+	def sprinth(hash={})
+		raise ArgumentError.new("sprinth argument must be a Hash") unless hash.is_a?(Hash)
+		self.gsub(/\%\w+\$/,"%") % self.scan(/\%\w+\$/).collect { |token| hash[token[1..-2]] }
+	end
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: rec
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Richard Kernahan
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-09-06 00:00:00 +10:00
+default_executable: 
+dependencies: []
+
+description: "\tSifts through your log files in real time, using stateful intelligence to determine\n\
+  \twhat is really important. REC can alert you (by email or IM) or it can simply condense\n\
+  \ta large log file into a much shorter and more meaningful log.\n\
+  \tREC is inspired by Risto Vaarandi's brilliant *sec* (simple-evcorr.sourceforge.net)\n\
+  \tbut is original code and any defects are entirely mine.\n\
+  \tWhile event correlation is inherently complex, REC attempts to make common tasks easy\n\
+  \twhile preserving plenty of power and flexibility for ambitious tasks.\n"
+email: rec@finalstep.com.au
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/rec.rb
+- lib/rec/rule.rb
+- lib/rec/state.rb
+- lib/rec/correlator.rb
+- lib/rec/alert.rb
+- lib/rec/mock-alert.rb
+- lib/string.rb
+has_rdoc: true
+homepage: http://rubygems.org/gems/rec
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: rec
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Ruby event correlation
+test_files: []
+