notifu 1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/notifu +15 -0
- data/lib/notifu.rb +12 -0
- data/lib/notifu/actors/gammu_sms_bridge.rb +41 -0
- data/lib/notifu/actors/pagerduty.rb +0 -0
- data/lib/notifu/actors/slack_chan.rb +29 -0
- data/lib/notifu/actors/slack_msg.rb +29 -0
- data/lib/notifu/actors/smtp.rb +73 -0
- data/lib/notifu/actors/stdout.rb +16 -0
- data/lib/notifu/actors/twilio_call.rb +27 -0
- data/lib/notifu/cli.rb +13 -0
- data/lib/notifu/cli/object.rb +37 -0
- data/lib/notifu/cli/service.rb +53 -0
- data/lib/notifu/config.rb +120 -0
- data/lib/notifu/logger.rb +83 -0
- data/lib/notifu/mixins.rb +49 -0
- data/lib/notifu/model.rb +5 -0
- data/lib/notifu/model/contact.rb +15 -0
- data/lib/notifu/model/event.rb +52 -0
- data/lib/notifu/model/group.rb +14 -0
- data/lib/notifu/model/issue.rb +51 -0
- data/lib/notifu/model/sla.rb +20 -0
- data/lib/notifu/sensu/handler.rb +105 -0
- data/lib/notifu/util.rb +9 -0
- data/lib/notifu/workers/actor.rb +60 -0
- data/lib/notifu/workers/processor.rb +444 -0
- data/lib/notifu/workers/sidekiq_init.rb +24 -0
- metadata +90 -0
@@ -0,0 +1,83 @@
|
|
1
|
+
module Notifu
|
2
|
+
class Logger
|
3
|
+
|
4
|
+
attr_reader :syslog_enabled
|
5
|
+
attr_reader :elasticsearch_enabled
|
6
|
+
attr_reader :es
|
7
|
+
attr_reader :syslog
|
8
|
+
|
9
|
+
LEVELS = [ "debug",
|
10
|
+
"info",
|
11
|
+
"notice",
|
12
|
+
"warning",
|
13
|
+
"error",
|
14
|
+
"critical",
|
15
|
+
"alert",
|
16
|
+
"emergency" ]
|
17
|
+
|
18
|
+
def initialize (mod)
|
19
|
+
@syslog_enabled = Notifu::CONFIG[:logging][:syslog][:enabled]
|
20
|
+
@elasticsearch_enabled = Notifu::CONFIG[:logging][:elasticsearch][:enabled]
|
21
|
+
|
22
|
+
@logger = Log4r::Logger.new 'notifu'
|
23
|
+
|
24
|
+
if self.syslog_enabled
|
25
|
+
begin
|
26
|
+
@logger.outputters = Log4r::SyslogOutputter.new "notifu", ident: "notifu-#{mod}"
|
27
|
+
log "info", "Syslog socket opened"
|
28
|
+
rescue
|
29
|
+
@logger.outputters = Log4r::Outputter.stdout
|
30
|
+
log "error", "Failed to open local syslog socket, using STDOUT"
|
31
|
+
end
|
32
|
+
else
|
33
|
+
log "info", "Syslog disabled"
|
34
|
+
@logger.outputters = Log4r::Outputter.stdout
|
35
|
+
end
|
36
|
+
|
37
|
+
if self.elasticsearch_enabled
|
38
|
+
begin
|
39
|
+
@es = Elasticsearch::Client.new hosts: Notifu::CONFIG[:logging][:elasticsearch][:conn], retry_on_failure: false, transport_options: { request: { timeout: Notifu::CONFIG[:logging][:elasticsearch][:timeout] || 10 } }
|
40
|
+
log "info", "Action log output to ElasticSearch - " + Notifu::CONFIG[:logging][:elasticsearch][:conn].to_json
|
41
|
+
rescue
|
42
|
+
@es = false
|
43
|
+
log "error", "Failed to connect to ElasticSearch"
|
44
|
+
exit 1
|
45
|
+
end
|
46
|
+
else
|
47
|
+
log "info", "ElasticSearch action logging disabled"
|
48
|
+
@es = false
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
def action_log (type, event)
|
54
|
+
if self.elasticsearch_enabled && self.es
|
55
|
+
index_name = "notifu-" + Time.now.strftime("%Y.%m.%d").to_s
|
56
|
+
begin
|
57
|
+
self.es.index index: index_name, type: type, body: event
|
58
|
+
rescue Faraday::TimeoutError
|
59
|
+
log "error", "Action log action failed: ElasticSearch timeout"
|
60
|
+
log "info", "Action log: (#{type}) #{event.to_json}"
|
61
|
+
end
|
62
|
+
else
|
63
|
+
log "debug", "Action log: #{type}"
|
64
|
+
log "debug", "Action log: (#{type}) #{event.to_json}"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def log (prio, msg)
|
69
|
+
@logger.send prio.to_sym, msg
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
|
74
|
+
|
75
|
+
# class LogFormatter < Log4r::Formatter
|
76
|
+
# def format(event)
|
77
|
+
# event.to_yaml
|
78
|
+
# end
|
79
|
+
# end
|
80
|
+
|
81
|
+
# "#{time.utc.iso8601(3)} notifu-sidekiq[#{::Process.pid}]: #{context} (TID-#{Thread.current.object_id.to_s(36)}) #{severity}: #{message}\n"
|
82
|
+
|
83
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
class Object
|
2
|
+
def deep_symbolize_keys
|
3
|
+
return self.inject({}){|memo,(k,v) | memo[k.to_sym] = v.deep_symbolize_keys; memo} if self.is_a? Hash
|
4
|
+
return self.inject([]){|memo,v | memo << v.deep_symbolize_keys; memo} if self.is_a? Array
|
5
|
+
return self
|
6
|
+
end
|
7
|
+
end
|
8
|
+
class Numeric
|
9
|
+
def duration
|
10
|
+
secs = self.to_int
|
11
|
+
mins = secs / 60
|
12
|
+
hours = mins / 60
|
13
|
+
days = hours / 24
|
14
|
+
|
15
|
+
if days > 0
|
16
|
+
"#{days}d, #{hours % 24}h"
|
17
|
+
elsif hours > 0
|
18
|
+
"#{hours}h, #{mins % 60}min"
|
19
|
+
elsif mins > 0
|
20
|
+
"#{mins}min, #{secs % 60}s"
|
21
|
+
elsif secs >= 0
|
22
|
+
"#{secs}s"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_state
|
27
|
+
case self.to_int
|
28
|
+
when 0
|
29
|
+
return "OK"
|
30
|
+
when 1
|
31
|
+
return "WARNING"
|
32
|
+
when 2
|
33
|
+
return "CRITICAL"
|
34
|
+
else
|
35
|
+
return "UNKNOWN [#{self.to_s}]"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class String
|
41
|
+
def camelize
|
42
|
+
return self if self !~ /_/ && self =~ /[A-Z]+.*/
|
43
|
+
split('_').map{|e| e.capitalize}.join
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_state
|
47
|
+
return self.to_i.to_state
|
48
|
+
end
|
49
|
+
end
|
data/lib/notifu/model.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Model
|
3
|
+
class Event
|
4
|
+
include Notifu::Util
|
5
|
+
attr_reader :notifu_id
|
6
|
+
attr_reader :host
|
7
|
+
attr_reader :address
|
8
|
+
attr_reader :service
|
9
|
+
attr_reader :occurrences_trigger
|
10
|
+
attr_reader :occurrences_count
|
11
|
+
attr_reader :interval
|
12
|
+
attr_reader :time_last_event
|
13
|
+
attr_reader :time_last_notified
|
14
|
+
attr_reader :time_created
|
15
|
+
attr_reader :sgs
|
16
|
+
attr_reader :action
|
17
|
+
attr_reader :code
|
18
|
+
attr_reader :aspiring_code
|
19
|
+
attr_reader :message
|
20
|
+
attr_reader :api_endpoint
|
21
|
+
attr_reader :duration
|
22
|
+
attr_reader :unsilence
|
23
|
+
attr_reader :refresh
|
24
|
+
attr_accessor :process_result
|
25
|
+
|
26
|
+
def initialize args
|
27
|
+
payload = args.first
|
28
|
+
payload.each { |name, value| instance_variable_set("@#{name}", value) }
|
29
|
+
@time_last_notified = Hash.new.to_json.to_s
|
30
|
+
@time_created = self.time_last_event
|
31
|
+
@aspiring_code = self.code
|
32
|
+
@occurrences_trigger ||= 1
|
33
|
+
@refresh ||= nil
|
34
|
+
@unsilence ||= true
|
35
|
+
end
|
36
|
+
|
37
|
+
def group_sla
|
38
|
+
self.sgs.map { |gs| Hash[:group, gs.split(':')[0], :sla, gs.split(':')[1]] }
|
39
|
+
end
|
40
|
+
|
41
|
+
def data
|
42
|
+
@data ||= Hash[ instance_variables.map { |var| [var.to_s.sub(/^@/,""), instance_variable_get(var)] } ]
|
43
|
+
end
|
44
|
+
|
45
|
+
def update_process_result!(obj)
|
46
|
+
self.process_result ||= JSON.generate(Array.new)
|
47
|
+
self.process_result = JSON.generate(JSON.parse(self.process_result) + [ obj ])
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Model
|
3
|
+
class Issue < Ohm::Model
|
4
|
+
include Notifu::Util
|
5
|
+
attribute :notifu_id
|
6
|
+
attribute :host
|
7
|
+
attribute :address
|
8
|
+
attribute :service
|
9
|
+
attribute :occurrences_trigger
|
10
|
+
attribute :occurrences_count
|
11
|
+
attribute :interval
|
12
|
+
attribute :refresh
|
13
|
+
attribute :time_last_event
|
14
|
+
attribute :time_last_notified
|
15
|
+
attribute :time_created
|
16
|
+
attribute :sgs
|
17
|
+
attribute :action
|
18
|
+
attribute :code
|
19
|
+
attribute :aspiring_code
|
20
|
+
attribute :message
|
21
|
+
attribute :process_result
|
22
|
+
attribute :api_endpoint
|
23
|
+
attribute :duration
|
24
|
+
attribute :unsilence
|
25
|
+
index :notifu_id
|
26
|
+
index :host
|
27
|
+
index :service
|
28
|
+
unique :notifu_id
|
29
|
+
|
30
|
+
|
31
|
+
def time_last_notified? (group_name, sla_name)
|
32
|
+
begin
|
33
|
+
JSON.parse(self.time_last_notified)["#{group_name}:#{sla_name}"]
|
34
|
+
rescue
|
35
|
+
0
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def time_last_notified! (group_name, sla_name, time)
|
40
|
+
obj = JSON.parse(self.time_last_notified)
|
41
|
+
self.time_last_notified = JSON.generate(obj.merge({ "#{group_name}:#{sla_name}" => time }))
|
42
|
+
end
|
43
|
+
|
44
|
+
def create_from_event event
|
45
|
+
event.each { |name, value| instance_variable_set(name, value) }
|
46
|
+
self.save
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Notifu
|
2
|
+
module Model
|
3
|
+
class Sla < Ohm::Model
|
4
|
+
|
5
|
+
attribute :name
|
6
|
+
attribute :timeranges
|
7
|
+
attribute :refresh
|
8
|
+
attribute :actors
|
9
|
+
index :name
|
10
|
+
unique :name
|
11
|
+
|
12
|
+
def timerange_values (now=Time.now)
|
13
|
+
dow = now.wday - 1
|
14
|
+
dow = 6 if dow < 0
|
15
|
+
JSON.parse(self.timeranges)[dow]
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'sensu/redis'
|
2
|
+
require 'digest'
|
3
|
+
require 'multi_json'
|
4
|
+
|
5
|
+
module Sensu::Extension
|
6
|
+
class Notifu < Handler
|
7
|
+
|
8
|
+
def definition
|
9
|
+
{
|
10
|
+
type: 'extension',
|
11
|
+
name: 'notifu'
|
12
|
+
}
|
13
|
+
end
|
14
|
+
|
15
|
+
def name
|
16
|
+
definition[:name]
|
17
|
+
end
|
18
|
+
|
19
|
+
def options
|
20
|
+
return @options if @options
|
21
|
+
@options = {
|
22
|
+
:host => '127.0.0.1',
|
23
|
+
:port => 6379,
|
24
|
+
:db => 2
|
25
|
+
}
|
26
|
+
if @settings[:notifu].is_a?(Hash)
|
27
|
+
@options.merge!(@settings[:notifu])
|
28
|
+
end
|
29
|
+
@options
|
30
|
+
end
|
31
|
+
|
32
|
+
def description
|
33
|
+
'Notifu handler for Sensu Server'
|
34
|
+
end
|
35
|
+
|
36
|
+
def post_init
|
37
|
+
|
38
|
+
if @redis
|
39
|
+
yield(@redis)
|
40
|
+
else
|
41
|
+
Sensu::Redis.connect(options) do |connection|
|
42
|
+
connection.auto_reconnect = false
|
43
|
+
connection.reconnect_on_error = true
|
44
|
+
connection.on_error do |error|
|
45
|
+
@logger.warn(error)
|
46
|
+
end
|
47
|
+
@redis = connection
|
48
|
+
end
|
49
|
+
end
|
50
|
+
@redis.sadd("queues", "processor")
|
51
|
+
end
|
52
|
+
|
53
|
+
def run(event_data)
|
54
|
+
event = MultiJson.load(event_data, { :symbolize_keys => true })
|
55
|
+
notifu_id = Digest::SHA256.hexdigest("#{event[:client][:name]}:#{event[:client][:address]}:#{event[:check][:name]}").to_s[-10,10]
|
56
|
+
|
57
|
+
if event[:check][:name] == "keepalive"
|
58
|
+
sgs = event[:client][:sgs]
|
59
|
+
sgs ||= event[:client][:sla]
|
60
|
+
else
|
61
|
+
sgs = event[:check][:sgs]
|
62
|
+
sgs ||= event[:check][:sla]
|
63
|
+
end
|
64
|
+
|
65
|
+
payload = {
|
66
|
+
notifu_id: notifu_id,
|
67
|
+
host: event[:client][:name],
|
68
|
+
address: event[:client][:address],
|
69
|
+
service: event[:check][:name],
|
70
|
+
occurrences_trigger: event[:check][:occurrences],
|
71
|
+
occurrences_count: event[:occurrences],
|
72
|
+
interval: event[:check][:interval] || 0,
|
73
|
+
time_last_event: event[:check][:executed],
|
74
|
+
sgs: sgs,
|
75
|
+
action: event[:action],
|
76
|
+
code: event[:check][:status],
|
77
|
+
message: event[:check][:output],
|
78
|
+
duration: event[:check][:duration],
|
79
|
+
api_endpoint: "http://" + @settings[:api][:host].to_s + ":" + @settings[:api][:port].to_s
|
80
|
+
}
|
81
|
+
|
82
|
+
job = {
|
83
|
+
'class' => 'Notifu::Processor',
|
84
|
+
'args' => [ payload ],
|
85
|
+
'jid' => SecureRandom.hex(12),
|
86
|
+
'retry' => true,
|
87
|
+
'enqueued_at' => Time.now.to_f
|
88
|
+
}
|
89
|
+
|
90
|
+
begin
|
91
|
+
@redis.lpush("queue:processor", MultiJson.dump(job))
|
92
|
+
rescue Exception => e
|
93
|
+
yield "failed to send event to Notifu #{e.message}", 1
|
94
|
+
end
|
95
|
+
|
96
|
+
yield "sent event to Notifu #{notifu_id}", 0
|
97
|
+
end
|
98
|
+
|
99
|
+
def stop
|
100
|
+
yield
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
data/lib/notifu/util.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
require_relative "sidekiq_init"
|
2
|
+
|
3
|
+
$logger = Notifu::Logger.new 'actor'
|
4
|
+
|
5
|
+
Sidekiq.configure_server do |config|
|
6
|
+
config.redis = { url: Notifu::CONFIG[:redis_queues] }
|
7
|
+
Sidekiq::Logging.logger = Log4r::Logger.new 'sidekiq'
|
8
|
+
Sidekiq::Logging.logger.outputters = Log4r::SyslogOutputter.new 'sidekiq', ident: 'notifu-actor'
|
9
|
+
# Sidekiq::Logging.logger.formatter = Notifu::LogFormatter.new
|
10
|
+
Sidekiq::Logging.logger.level = Log4r::DEBUG
|
11
|
+
end
|
12
|
+
|
13
|
+
Sidekiq.configure_client do |config|
|
14
|
+
config.redis = { url: Notifu::CONFIG[:redis_queues] }
|
15
|
+
end
|
16
|
+
|
17
|
+
|
18
|
+
module Notifu
|
19
|
+
|
20
|
+
class Actor
|
21
|
+
include Notifu::Util
|
22
|
+
include Sidekiq::Worker
|
23
|
+
|
24
|
+
attr_accessor :issue
|
25
|
+
attr_accessor :contacts
|
26
|
+
|
27
|
+
class << self
|
28
|
+
attr_accessor :name
|
29
|
+
attr_accessor :desc
|
30
|
+
attr_accessor :retry
|
31
|
+
end
|
32
|
+
|
33
|
+
# `act` function must be defined in child classes
|
34
|
+
# (provides notification actors modularity)
|
35
|
+
#
|
36
|
+
def act
|
37
|
+
exit 1
|
38
|
+
end
|
39
|
+
|
40
|
+
sidekiq_options :queue => "actor-#{self.name}"
|
41
|
+
sidekiq_options :retry => self.retry
|
42
|
+
|
43
|
+
def perform *args
|
44
|
+
sleep 2
|
45
|
+
load_data args
|
46
|
+
act
|
47
|
+
end
|
48
|
+
|
49
|
+
def load_data args
|
50
|
+
self.issue = Notifu::Model::Issue.with(:notifu_id, args[0])
|
51
|
+
self.contacts = args[1].map { |contact| Notifu::Model::Contact.with(:name, contact) }
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
# load all actors
|
58
|
+
Dir[File.dirname(__FILE__).sub(/\/lib\/workers$/, "/") + 'actors/*.rb'].each do |file|
|
59
|
+
require file
|
60
|
+
end
|