systemd_mon_mod 0.1.1

data/lib/systemd_mon/notification_centre.rb

@@ -0,0 +1,77 @@
+require 'systemd_mon/error'
+require 'systemd_mon/logger'
+
+module SystemdMon
+  class NotificationCentre
+    include Enumerable
+
+    def initialize
+      self.notifiers = []
+    end
+
+    def classes
+      notifiers.map(&:class)
+    end
+
+    def each
+      notifiers.each do |notifier|
+        yield notifier
+      end
+    end
+
+    def add_notifier(notifier)
+      unless notifier.respond_to?(:notify!)
+        raise NotifierError, "Notifier #{notifier.class} must respond to 'notify!'"
+      end
+      self.notifiers << notifier
+    end
+
+    def notify_start!(hostname)
+      each_notifier do |notifier|
+        if notifier.respond_to?(:notify_start!)
+          Logger.puts "Notifying SystemdMon start via #{notifier.class}"
+          notifier.notify_start! hostname
+        else
+          Logger.debug { "#{notifier.class} doesn't respond to 'notify_start!', not sending notification" }
+        end
+      end
+    end
+
+    def notify_stop!(hostname)
+      each_notifier do |notifier|
+        if notifier.respond_to?(:notify_stop!)
+          Logger.puts "Notifying SystemdMon stop via #{notifier.class}"
+          notifier.notify_stop! hostname
+        else
+          Logger.debug { "#{notifier.class} doesn't respond to 'notify_start!', not sending notification" }
+        end
+      end
+    end
+
+    def notify!(notification)
+      each_notifier do |notifier|
+        Logger.puts "Notifying state change of #{notification.unit.name} via #{notifier.class}"
+        notifier.notify! notification
+      end
+    end
+
+    alias :<< :add_notifier
+
+  protected
+    attr_accessor :notifiers
+
+    def each_notifier
+      notifiers.map { |notifier|
+        Thread.new do
+          begin
+            yield notifier
+          rescue => e
+            Logger.error "Failed to send notification via #{notifier.class}:\n"
+            Logger.error "  #{e.class}: #{e.message}\n"
+            Logger.debug_error { "\n\t#{e.backtrace.join('\n\t')}\n" }
+          end
+        end
+      }.each(&:join)
+    end
+  end
+end

data/lib/systemd_mon/notifier_loader.rb

@@ -0,0 +1,21 @@
+module SystemdMon
+  class NotifierLoader
+    def get_class(name)
+      class_name = camel_case(name)
+      get_class_const(class_name)
+    rescue NameError
+      require "systemd_mon/notifiers/#{name}"
+      get_class_const(class_name)
+    end
+
+  protected
+    def camel_case(name)
+      return name if name !~ /_/ && name =~ /[A-Z]+.*/
+      name.split('_').map { |e| e.capitalize }.join
+    end
+
+    def get_class_const(name)
+      Notifiers.const_get(name)
+    end
+  end
+end

data/lib/systemd_mon/notifiers/base.rb

@@ -0,0 +1,39 @@
+require 'systemd_mon/logger'
+
+module SystemdMon::Notifiers
+  class Base
+    def initialize(options)
+      self.options = options
+      self.me      = self.class.name
+    end
+
+    # Subclasses must respond to a unit change
+    def notify!(notification)
+      raise "Notifier #{self.class} does not respond to notify!"
+    end
+
+    # Subclasses can choose to do something when SystemdMon starts
+    # E.g. with
+    #
+    # def notify_start!(hostname)
+    # end
+
+    # Subclasses can choose to do something when SystemdMon stops
+    # E.g. with
+    #
+    # def notify_stop!(hostname)
+    # end
+
+    def log(message)
+      SystemdMon::Logger.puts "#{me}: #{message}"
+    end
+
+    def debug(message = nil, &blk)
+      message = "#{me}: #{message}" if message
+      SystemdMon::Logger.debug message, &blk
+    end
+
+  protected
+    attr_accessor :options, :me
+  end
+end

data/lib/systemd_mon/notifiers/email.rb

@@ -0,0 +1,65 @@
+require 'systemd_mon/error'
+require 'systemd_mon/notifiers/base'
+require 'systemd_mon/formatters/state_table_formatter'
+
+begin
+  require 'mail'
+rescue LoadError
+  raise SystemdMon::NotifierDependencyError, "The 'mail' gem is required by the email notifier"
+end
+
+module SystemdMon::Notifiers
+  class Email < Base
+    def initialize(*)
+      super
+      if options['smtp']
+        opts = options
+        Mail.defaults do
+          delivery_method :smtp, Hash[opts['smtp'].map { |h, k| [h.to_sym, k] }]
+        end
+      end
+
+      validate_options!
+    end
+
+    def notify!(notification)
+      unit = notification.unit
+      subject = "#{notification.type_text}: #{unit.name} on #{notification.hostname}: #{unit.state_change.status_text}"
+      message = "Systemd unit #{unit.name} on #{notification.hostname} #{unit.state_change.status_text}: #{unit.state.active} (#{unit.state.sub})\n\n"
+      if unit.state_change.length > 1
+        message << SystemdMon::Formatters::StateTableFormatter.new(unit).as_text
+      end
+      message << "\nRegards, SystemdMon"
+
+      send_mail subject, message
+
+      log "sent email notification"
+    end
+
+  protected
+    attr_accessor :options
+
+    def validate_options!
+      unless options.has_key?("to")
+        raise NotifierError, "The 'to' address must be set to use the email notifier"
+      end
+      true
+    end
+
+    def send_mail(subject, message)
+      debug("Sending email to #{options['to']}:")
+      debug(%Q{ -> Subject: "#{subject}"})
+      debug(%Q{ -> Message: "#{message}"})
+
+      mail = Mail.new do
+        subject subject
+        body message
+      end
+      mail.to = options['to']
+      if options['from']
+        mail.from options['from']
+      end
+      mail.deliver!
+    end
+  end
+end

data/lib/systemd_mon/notifiers/hipchat.rb

@@ -0,0 +1,62 @@
+require 'systemd_mon/error'
+require 'systemd_mon/notifiers/base'
+
+begin
+  require 'hipchat'
+rescue LoadError
+  raise SystemdMon::NotifierDependencyError, "The 'hipchat' gem is required by the hipchat notifier"
+end
+
+module SystemdMon::Notifiers
+  class Hipchat < Base
+    def initialize(*)
+      super
+      self.client = ::HipChat::Client.new(
+          options['token'],
+          :api_version => 'v2')
+    end
+
+    def notify_start!(hostname)
+      chat "SystemdMon is starting on #{hostname}",
+           'green'
+    end
+
+    def notify_stop!(hostname)
+      chat "SystemdMon is stopping on #{hostname}",
+           'yellow'
+    end
+
+    def notify!(notification)
+      unit = notification.unit
+      message = "#{notification.type_text}: systemd unit #{unit.name} on #{notification.hostname} #{unit.state_change.status_text}: #{unit.state.active} (#{unit.state.sub})"
+
+      chat message,
+           color(notification.type)
+
+      log "sent hipchat notification"
+    end
+
+  protected
+    attr_accessor :client, :options
+
+    def chat(message, shade)
+      client[options['room']].send(
+        options['username'],
+        message,
+        :color => shade)
+    end
+
+    def color(type)
+      case type
+      when :alert
+        'red'
+      when :warning
+        'yellow'
+      when :info
+        'purple'
+      else
+        'green'
+      end
+    end
+  end
+end

data/lib/systemd_mon/notifiers/slack.rb

@@ -0,0 +1,99 @@
+require 'systemd_mon/error'
+require 'systemd_mon/notifiers/base'
+
+begin
+  require 'slack-notifier'
+rescue LoadError
+  raise SystemdMon::NotifierDependencyError, "The 'slack-notifier' gem (> 1.0) is required by the slack notifier"
+end
+
+module SystemdMon::Notifiers
+  class Slack < Base
+    def initialize(*)
+      super
+      self.notifier = ::Slack::Notifier.new(
+        options.fetch('webhook_url'),
+        channel: options['channel'],
+        username: options['username'],
+        icon_emoji: options['icon_emoji'],
+        icon_url: options['icon_url'])
+    end
+
+    def notify_start!(hostname)
+      message = "@channel SystemdMon is starting on #{hostname}"
+
+      attach = {
+        fallback: message,
+        text: message,
+        color: "good"
+      }
+
+      notifier.ping "", attachments: [attach]
+    end
+
+    def notify_stop!(hostname)
+      message = "@channel SystemdMon is stopping on #{hostname}"
+
+      attach = {
+        fallback: message,
+        text: message,
+        color: "danger"
+      }
+
+      notifier.ping "", attachments: [attach]
+    end
+
+    def notify!(notification)
+      unit = notification.unit
+      message = "@channel #{notification.type_text}: systemd unit #{unit.name} on #{notification.hostname} #{unit.state_change.status_text}"
+
+      attach = {
+        fallback: "@channel #{message}: #{unit.state.active} (#{unit.state.sub})",
+        color: color(notification.type),
+        fields: fields(notification)
+      }
+
+      debug("sending slack message with attachment: ")
+      debug(attach.inspect)
+
+      notifier.ping message, attachments: [attach]
+      log "sent slack notification"
+    end
+
+  protected
+    attr_accessor :notifier
+
+    def fields(notification)
+      f = [
+        {
+          title: "Hostname",
+          value: notification.hostname,
+          short: true
+        },
+        {
+          title: "Unit",
+          value: notification.unit.name,
+          short: true
+        }
+      ]
+
+      changes = notification.unit.state_change.diff.map(&:last)
+      f.concat(changes.map { |v|
+        { title: v.display_name, value: v.value, short: true }
+      })
+    end
+
+    def color(type)
+      case type
+      when :alert
+        'danger'
+      when :warning
+        '#FF9900'
+      when :info
+        '#0099CC'
+      else
+        'good'
+      end
+    end
+  end
+end

data/lib/systemd_mon/state.rb

@@ -0,0 +1,59 @@
+require 'systemd_mon/state_value'
+
+module SystemdMon
+  class State
+    include Enumerable
+
+    attr_reader :active, :sub, :loaded, :unit_file, :all_states
+
+    def initialize(active, sub, loaded, unit_file, type=nil)
+      timestamp   = Time.now
+      @active     = StateValue.new("active", active, timestamp, *active_states(type))
+      @sub        = StateValue.new("status", sub, timestamp)
+      @loaded     = StateValue.new("loaded", loaded, timestamp, %w(loaded))
+      @unit_file  = StateValue.new("file", unit_file, timestamp, *file_states(type))
+      @all_states = [@active, @sub, @loaded, @unit_file]
+    end
+
+    def active_states(type)
+      case type
+      when 'oneshot'
+        [%w(inactive), %w(failed)]
+      else
+        [%w(active), %w(inactive failed)]
+      end
+    end
+
+    def file_states(type)
+      case type
+      when 'oneshot'
+        [[], []]
+      else
+        [%w(enabled linked-runtime static), %w(disabled)]
+      end
+    end
+
+    def each
+      @all_states.each do |state|
+        yield state
+      end
+    end
+
+    def ok?
+      all?(&:ok?)
+    end
+
+    def fail?
+      any?(&:fail?)
+    end
+
+    def to_s
+      @all_states.join(', ')
+    end
+
+    def ==(other)
+      @all_states == other.all_states
+    end
+  end
+
+end

data/lib/systemd_mon/state_change.rb

@@ -0,0 +1,119 @@
+module SystemdMon
+  class StateChange
+    include Enumerable
+
+    attr_reader :states
+
+    def initialize(original_state = nil)
+      self.states = []
+      states << original_state if original_state
+    end
+
+    def last
+      states.last
+    end
+
+    def length
+      states.length
+    end
+
+    def <<(state)
+      self.states << state
+      @diff = nil
+    end
+
+    def each
+      states.each do |state|
+        yield state
+      end
+    end
+
+    def changes
+      states[1..-1]
+    end
+
+    def recovery?
+      first.fail? && last.ok?
+    end
+
+    def ok?
+      last.ok?
+    end
+
+    def fail?
+      last.fail?
+    end
+
+    def restart?
+      first.ok? && last.ok? && changes.any? { |s| s.active == "deactivating" }
+    end
+
+    def auto_restart?
+      first.ok? && last.ok? && changes.any? { |s| s.sub == "auto-restart" }
+    end
+
+    def reload?
+      first.ok? && last.ok? && changes.any? { |s| s.active == "reloading" }
+    end
+
+    def still_fail?
+      length > 1 && first.fail? && last.fail?
+    end
+
+    def status_text
+      if recovery?
+        "recovered"
+      elsif auto_restart?
+        "automatically restarted"
+      elsif restart?
+        "restarted"
+      elsif reload?
+        "reloaded"
+      elsif still_fail?
+        "still failed"
+      elsif fail?
+        "failed"
+      else
+        "started"
+      end
+    end
+
+    def important?
+      if length == 1
+        first.fail?
+      else
+        diff.map(&:last).any?(&:important?)
+      end
+    end
+
+    def diff
+      @diff ||= zipped.reject { |states|
+        match = states.first.value
+        states.all? { |s| s.value == match }
+      }
+    end
+
+    def zipped
+      if length == 1
+        first.all_states
+      else
+        first.all_states.zip(*changes.map(&:all_states))
+      end
+    end
+
+    def to_s
+      diff.inject("") { |s, (*states)|
+        first = states.shift
+        s << "#{first.name} state changed from #{first.value} to "
+        s << states.map(&:value).join(" then ")
+        s << "\n"
+        s
+      }
+    end
+
+  protected
+    attr_accessor :original, :changed
+    attr_writer :states
+  end
+
+end

data/lib/systemd_mon/state_value.rb

@@ -0,0 +1,48 @@
+module SystemdMon
+  class StateValue
+    attr_reader :name, :value, :ok_states, :failure_states, :timestamp
+
+    def initialize(name, value, timestamp, ok_states = [], failure_states = [])
+      self.name = name
+      self.value = value
+      self.ok_states = ok_states
+      self.failure_states = failure_states
+      self.timestamp = timestamp
+    end
+
+    def display_name
+      name.capitalize
+    end
+
+    def important?
+      ok_states.include?(value) || failure_states.include?(value)
+    end
+
+    def ok?
+      if ok_states.any?
+        ok_states.include?(value)
+      else
+        true
+      end
+    end
+
+    def fail?
+      if failure_states.any?
+        failure_states.include?(value)
+      else
+        false
+      end
+    end
+
+    def to_s
+      value
+    end
+
+    def ==(other)
+      other.is_a?(SystemdMon::StateValue) && value == other.value
+    end
+
+  protected
+    attr_writer :name, :value, :ok_states, :failure_states, :timestamp
+  end
+end

data/lib/systemd_mon/unit_with_state.rb

@@ -0,0 +1,33 @@
+require 'systemd_mon/state_change'
+
+module SystemdMon
+  class UnitWithState
+    attr_reader :unit, :state_change
+
+    def initialize(unit)
+      self.unit         = unit
+      self.state_change = StateChange.new
+    end
+
+    def name
+      unit.name
+    end
+
+    def <<(state)
+      self.state_change << state
+    end
+
+    def current_state
+      state_change.last
+    end
+
+    def reset!
+      self.state_change = StateChange.new(current_state)
+    end
+
+    alias :state :current_state
+
+  protected
+    attr_writer :state_change, :unit
+  end
+end

data/lib/systemd_mon/version.rb

@@ -0,0 +1,3 @@
+module SystemdMon
+  VERSION = "0.1.1"
+end

data/lib/systemd_mon.rb

@@ -0,0 +1,5 @@
+require "systemd_mon/version"
+require "systemd_mon/logger"
+require "systemd_mon/notifier_loader"
+
+module SystemdMon; end

data/systemd_mon.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'systemd_mon/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "systemd_mon_mod"
+  spec.version       = SystemdMon::VERSION
+  spec.authors       = ["Jon Cairns", "Dmitriy Maksakov"]
+  spec.email         = ["jon@joncairns.com", "mksvdmtr@yandex.ru"]
+  spec.summary       = %q{Monitor systemd units and trigger alerts for failed states (Mod for mattermost @channel mentioning)}
+  spec.description   = %q{Monitor systemd units and trigger alerts for failed states (Mod for mattermost @channel mentioning)}
+  spec.homepage      = "https://github.com/mksvdmtr/systemd_mon_mod"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "ruby-dbus", "~> 0.11.0"
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end