systemd_mon 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 71beedd875598eeb882df66a2a2982aad63fffe3
+  data.tar.gz: e08a8386c4489f8f42461ed8aa5bf2a65eb78bf6
+SHA512:
+  metadata.gz: 016247aaa8c34f2cb6eaddd14459bc58d52ccd12e32557ca3a5d4e5a30c06e5222db72393f5d630d293ec69533897d7a5d6fdd7d1a161bec2a02ae5acaf43ed2
+  data.tar.gz: ca44541313c92f37e790f15ba07862885288d1d4e026bdb1560f681f1bc3b3cda721ec6b608b18c6507a0b65cf763bc75bd039f010e6818243e020b71f28c29b

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in systemd_alert.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Jon Cairns
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# SystemdMon
+
+Monitor systemd units and trigger alerts for failed states. The command line tool runs as a daemon, using dbus to get notifications of changes to systemd services. If a service enters a failed state, or returns from a failed state to an active state, notifications will be triggered.
+
+Built-in notifications include email and slack, but more can be added via the ruby API.
+
+## Installation
+
+Install the gem using:
+
+    gem install systemd_mon
+
+## Usage
+
+To run the command line tool, you will first need to create a YAML configuration file to specify which systemd units you want to monitor, and which notifications you want to trigger. A full example looks like this:
+
+```yaml
+---
+notifiers:
+  # These are options passed to the 'mail' gem
+  email:
+    address: smtp.gmail.com
+    port: 587
+    domain: mydomain.com
+    user_name: "user@mydomain.com"
+    password: "supersecr3t"
+    authentication: "plain"
+    enable_starttls_auto: true
+  slack:
+    team: myteam
+    token: supersecr3ttoken
+    channel: mychannel
+    username: doge
+units:
+- unicorn.service
+- nginx.service
+- sidekiq.service
+```
+
+Then start the command line tool with:
+
+    $ systemd_mon path/to/systemd_mon.yml
+
+## Contributing
+
+1. Fork it ( https://github.com/joonty/systemd_mon/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/systemd_mon

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'systemd_mon/cli'
+
+SystemdMon::CLI.new.start

data/lib/systemd_mon.rb

@@ -0,0 +1,5 @@
+require "systemd_mon/version"
+require "systemd_mon/logger"
+require "systemd_mon/notifier_loader"
+
+module SystemdMon; end

data/lib/systemd_mon/callback_manager.rb

@@ -0,0 +1,39 @@
+require 'systemd_mon/unit_with_state'
+
+module SystemdMon
+  class CallbackManager
+    def initialize(queue)
+      self.queue  = queue
+      self.states = Hash.new { |h, u| h[u] = UnitWithState.new(u) }
+    end
+
+    def start(change_callback, each_state_change_callback)
+      loop do
+        unit, state = queue.deq
+        Logger.debug { state }
+        unit_state = states[unit]
+        unit_state << state
+
+        if each_state_change_callback
+          with_error_handling { each_state_change_callback.call(unit_state) }
+        end
+
+        if change_callback && unit_state.state_change.important?
+          with_error_handling { change_callback.call(unit_state) }
+        end
+
+        unit_state.reset! if unit_state.state_change.important?
+      end
+    end
+
+    def with_error_handling
+      yield
+    rescue => e
+      Logger.error "Uncaught exception (#{e.class}) in callback: #{e.message}"
+      Logger.debug_error { "\n\t#{e.backtrace.join("\n\t")}\n" }
+    end
+
+  protected
+    attr_accessor :queue, :states
+  end
+end

data/lib/systemd_mon/cli.rb

@@ -0,0 +1,81 @@
+require 'yaml'
+require 'systemd_mon'
+require 'systemd_mon/monitor'
+require 'systemd_mon/error'
+require 'systemd_mon/dbus_manager'
+
+module SystemdMon
+  class CLI
+    def initialize
+      self.me      = "systemd_mon"
+      self.verbose = true
+    end
+
+    def start
+      yaml_config_file = ARGV.first
+      self.options = load_and_validate_options(yaml_config_file)
+      self.verbose = options['verbose'] || false
+      Logger.verbose = verbose
+
+      start_monitor
+
+    rescue SystemdMon::Error => e
+      err_string = e.message
+      if verbose
+        err_string << " - #{e.original.message} (#{e.original.class})"
+        err_string << "\n\t#{e.original.backtrace.join("\n\t")}"
+      end
+      fatal_error(err_string)
+    rescue => e
+      err_string = e.message
+      if verbose
+        err_string << " (#{e.class})"
+        err_string << "\n\t#{e.backtrace.join("\n\t")}"
+      end
+      fatal_error(err_string)
+    end
+
+  protected
+    def start_monitor
+      monitor = Monitor.new(DBusManager.new)
+
+      # Load units to monitor
+      monitor.register_units options['units']
+
+      options['notifiers'].each do |name, notifier_options|
+        klass = NotifierLoader.new.get_class(name)
+        monitor.add_notifier klass.new(notifier_options)
+      end
+
+      monitor.start
+    end
+
+    def load_and_validate_options(yaml_config_file)
+      options = load_options(yaml_config_file)
+
+      unless options.has_key?('notifiers') && options['notifiers'].any?
+        fatal_error("no notifiers have been defined, there is no reason to continue")
+      end
+      unless options.has_key?('units') && options['units'].any?
+        fatal_error("no units have been added for watching, there is no reason to continue")
+      end
+      options
+    end
+
+    def load_options(yaml_config_file)
+      unless yaml_config_file && File.exists?(yaml_config_file)
+        fatal_error "First argument must be a path to a YAML configuration file"
+      end
+
+      YAML.load_file(yaml_config_file)
+    end
+
+    def fatal_error(message, code = 255)
+      $stderr.puts " #{me} error: #{message}"
+      exit code
+    end
+
+  protected
+    attr_accessor :verbose, :options, :me
+  end
+end

data/lib/systemd_mon/dbus_manager.rb

@@ -0,0 +1,31 @@
+require 'dbus'
+require 'systemd_mon/error'
+require 'systemd_mon/dbus_unit'
+
+module SystemdMon
+  class DBusManager
+    def initialize
+      self.dbus            = DBus::SystemBus.instance
+      self.systemd_service = dbus.service("org.freedesktop.systemd1")
+      self.systemd_object  = systemd_service.object("/org/freedesktop/systemd1")
+      systemd_object.introspect
+      systemd_object.Subscribe
+    end
+
+    def fetch_unit(unit_name)
+      path = systemd_object.GetUnit(unit_name).first
+      DBusUnit.new(unit_name, path, systemd_service.object(path))
+    rescue DBus::Error
+      raise SystemdMon::UnknownUnitError, "Unknown or unloaded systemd unit '#{unit_name}'"
+    end
+
+    def runner
+      main = DBus::Main.new
+      main << dbus
+      main
+    end
+
+  protected
+    attr_accessor :systemd_service, :systemd_object, :dbus
+  end
+end

data/lib/systemd_mon/dbus_unit.rb

@@ -0,0 +1,57 @@
+require 'systemd_mon/state'
+
+module SystemdMon
+  class DBusUnit
+    attr_reader :name
+
+    IFACE_UNIT  = "org.freedesktop.systemd1.Unit"
+    IFACE_PROPS = "org.freedesktop.DBus.Properties"
+
+    def initialize(name, path, dbus_object)
+      self.name = name
+      self.path = path
+      self.dbus_object = dbus_object
+      prepare_dbus_objects!
+    end
+
+    def register_listener!(queue)
+      queue.enq [self, build_state] # initial state
+      dbus_object.on_signal("PropertiesChanged") do |iface|
+        if iface == IFACE_UNIT
+          queue.enq [self, build_state]
+        end
+      end
+    end
+
+    def on_change(&callback)
+      self.change_callback = callback
+    end
+
+    def on_each_state_change(&callback)
+      self.each_state_change_callback = callback
+    end
+
+    def property(name)
+      dbus_object.Get(IFACE_UNIT, name).first
+    end
+
+  protected
+    attr_accessor :path, :dbus_object, :change_callback, :each_state_change_callback
+    attr_writer   :name
+
+    def build_state
+      State.new(
+        property("ActiveState"),
+        property("SubState"),
+        property("LoadState"),
+        property("UnitFileState")
+      )
+    end
+
+    def prepare_dbus_objects!
+      dbus_object.introspect
+      self.dbus_object.default_iface = IFACE_PROPS
+      self
+    end
+  end
+end

data/lib/systemd_mon/error.rb

@@ -0,0 +1,17 @@
+module SystemdMon
+
+  # Save original exception for use in verbose mode
+  class Error < StandardError
+    attr_reader :original
+
+    def initialize(msg, original=$!)
+      super(msg)
+      @original = original
+    end
+  end
+
+  class MonitorError < Error; end
+  class UnknownUnitError < Error; end
+  class NotificationError < Error; end
+  class NotifierError < Error; end
+end

data/lib/systemd_mon/formatters/base.rb

@@ -0,0 +1,18 @@
+module SystemdMon::Formatters
+  class Base
+    def initialize(unit)
+      self.unit = unit
+    end
+
+    def as_html
+      raise "The formatter #{self.class} does not provide an html formatted string"
+    end
+
+    def as_text
+      raise "The formatter #{self.class} does not provide a plain text string"
+    end
+
+  protected
+    attr_accessor :unit
+  end
+end

data/lib/systemd_mon/formatters/state_table_formatter.rb

@@ -0,0 +1,32 @@
+require 'systemd_mon/formatters/base'
+module SystemdMon::Formatters
+  class StateTableFormatter < Base
+    def as_text
+      table = render_table
+      lengths = table.transpose.map { |v| v.map(&:length).max }
+
+      full_width = lengths.inject(&:+) + (lengths.length * 3) + 1
+      div = " " + ("-" * full_width) + "\n"
+      s = div.dup
+      table.each do |row|
+        s << " | "
+        row.each_with_index { |col, i|
+          s << col.ljust(lengths[i]) + " | "
+        }
+        s << "\n" + div.dup
+      end
+      s
+    end
+
+  protected
+    def render_table
+      changed = unit.state_change.diff
+      table = []
+      table << ["Time"].concat(changed.map{|v| v.first.display_name})
+      changed.transpose.each do |vals|
+        table << [vals.first.timestamp.strftime("%H:%M:%S.%3N %z")].concat(vals.map{|v| v.value})
+      end
+      table
+    end
+  end
+end

data/lib/systemd_mon/logger.rb

@@ -0,0 +1,33 @@
+module SystemdMon
+  class Logger
+    def self.verbose=(flag)
+      @verbose = flag
+    end
+
+    def self.verbose
+      @verbose
+    end
+
+    def self.debug(message = nil, stream = $stdout)
+      if verbose
+        if block_given?
+          $stdout.puts yield
+        else
+          $stdout.puts message
+        end
+      end
+    end
+
+    def self.error(message = nil)
+      $stderr.puts message
+    end
+
+    def self.debug_error(message = nil)
+      debug message, $stderr
+    end
+
+    def self.puts(message = nil)
+      $stdout.puts message
+    end
+  end
+end

data/lib/systemd_mon/monitor.rb

@@ -0,0 +1,98 @@
+require 'thread'
+require 'systemd_mon/logger'
+require 'systemd_mon/callback_manager'
+require 'systemd_mon/notification_centre'
+require 'systemd_mon/notification'
+require 'systemd_mon/error'
+
+module SystemdMon
+  class Monitor
+    def initialize(dbus_manager)
+      self.hostname     = `hostname`.strip
+      self.dbus_manager = dbus_manager
+      self.units        = []
+      self.change_callback     = lambda(&method(:unit_change_callback))
+      self.notification_centre = NotificationCentre.new
+      Thread.abort_on_exception = true
+    end
+
+    def add_notifier(notifier)
+      notification_centre << notifier
+      self
+    end
+
+    def register_unit(unit_name)
+      self.units << dbus_manager.fetch_unit(unit_name)
+      self
+    end
+
+    def register_units(*unit_names)
+      self.units.concat unit_names.flatten.map { |unit_name|
+        dbus_manager.fetch_unit(unit_name)
+      }
+      self
+    end
+
+    def on_change(&callback)
+      self.change_callback = callback
+      self
+    end
+
+    def on_each_state_change(&callback)
+      self.each_state_change_callback = callback
+      self
+    end
+
+    def start
+      startup_check!
+      at_exit { notification_centre.notify_stop! hostname }
+      notification_centre.notify_start! hostname
+
+      Logger.puts "Monitoring changes to #{units.count} units"
+      Logger.debug { " - " + units.map(&:name).join("\n - ") + "\n\n" }
+      Logger.debug { "Using notifiers: #{notification_centre.classes.join(", ")}"}
+
+      state_q = Queue.new
+
+      units.each do |unit|
+        unit.register_listener! state_q
+      end
+
+      [start_callback_thread(state_q),
+       start_dbus_thread].each(&:join)
+    end
+
+protected
+    attr_accessor :units, :dbus_manager, :change_callback, :each_state_change_callback, :hostname, :notification_centre
+
+    def startup_check!
+      unless units.any?
+        raise MonitorError, "At least one systemd unit should be registered before monitoring can start"
+      end
+      unless notification_centre.any?
+        raise MonitorError, "At least one notifier should be registered before monitoring can start"
+      end
+      self
+    end
+
+    def start_dbus_thread
+      Thread.new do
+        dbus_manager.runner.run
+      end
+    end
+
+    def start_callback_thread(state_q)
+      Thread.new do
+        manager = CallbackManager.new(state_q)
+        manager.start change_callback, each_state_change_callback
+      end
+    end
+
+    def unit_change_callback(unit)
+      Logger.puts "#{unit.name} #{unit.state_change.status_text}: #{unit.state.active} (#{unit.state.sub})"
+      Logger.debug unit.state_change.to_s
+      Logger.puts
+      notification_centre.notify! Notification.new(hostname, unit)
+    end
+  end
+end

data/lib/systemd_mon/notification.rb

@@ -0,0 +1,30 @@
+module SystemdMon
+  class Notification
+    attr_reader :unit, :type, :hostname
+
+    def initialize(hostname, unit)
+      self.hostname = hostname
+      self.unit     = unit
+      self.type     = determine_type
+    end
+
+    def self.types
+      [:alert, :info, :ok]
+    end
+
+  protected
+    attr_writer :unit, :type, :hostname
+
+    def determine_type
+      if unit.state_change.ok?
+        if unit.state_change.first.fail?
+          :ok
+        else
+          :info
+        end
+      else
+        :alert
+      end
+    end
+  end
+end

data/lib/systemd_mon/notification_centre.rb

@@ -0,0 +1,77 @@
+require 'systemd_mon/error'
+require 'systemd_mon/logger'
+
+module SystemdMon
+  class NotificationCentre
+    include Enumerable
+
+    def initialize
+      self.notifiers = []
+    end
+
+    def classes
+      notifiers.map(&:class)
+    end
+
+    def each
+      notifiers.each do |notifier|
+        yield notifier
+      end
+    end
+
+    def add_notifier(notifier)
+      unless notifier.respond_to?(:notify!)
+        raise NotifierError, "Notifier #{notifier.class} must respond to 'notify!'"
+      end
+      self.notifiers << notifier
+    end
+
+    def notify_start!(hostname)
+      each_notifier do |notifier|
+        if notifier.respond_to?(:notify_start!)
+          Logger.puts "Notifying SystemdMon start via #{notifier.class}"
+          notifier.notify_start! hostname
+        else
+          Logger.debug { "#{notifier.class} doesn't respond to 'notify_start!', not sending notification" }
+        end
+      end
+    end
+
+    def notify_stop!(hostname)
+      each_notifier do |notifier|
+        if notifier.respond_to?(:notify_stop!)
+          Logger.puts "Notifying SystemdMon stop via #{notifier.class}"
+          notifier.notify_stop! hostname
+        else
+          Logger.debug { "#{notifier.class} doesn't respond to 'notify_start!', not sending notification" }
+        end
+      end
+    end
+
+    def notify!(notification)
+      each_notifier do |notifier|
+        Logger.puts "Notifying state change of #{notification.unit.name} via #{notifier.class}"
+        notifier.notify! notification
+      end
+    end
+
+    alias :<< :add_notifier
+
+  protected
+    attr_accessor :notifiers
+
+    def each_notifier
+      notifiers.map { |notifier|
+        Thread.new do
+          begin
+            yield notifier
+          rescue => e
+            Logger.error "Failed to send notification via #{notifier.class}:\n"
+            Logger.error "  #{e.class}: #{e.message}\n"
+            Logger.debug_error { "\n\t#{e.backtrace.join('\n\t')}\n" }
+          end
+        end
+      }.each(&:join)
+    end
+  end
+end

data/lib/systemd_mon/notifier_loader.rb

@@ -0,0 +1,21 @@
+module SystemdMon
+  class NotifierLoader
+    def get_class(name)
+      class_name = camel_case(name)
+      get_class_const(class_name)
+    rescue NameError
+      require "systemd_mon/notifiers/#{name}"
+      get_class_const(class_name)
+    end
+
+  protected
+    def camel_case(name)
+      return name if name !~ /_/ && name =~ /[A-Z]+.*/
+      name.split('_').map { |e| e.capitalize }.join
+    end
+
+    def get_class_const(name)
+      Notifiers.const_get(name)
+    end
+  end
+end

data/lib/systemd_mon/notifiers/base.rb

@@ -0,0 +1,39 @@
+require 'systemd_mon/logger'
+
+module SystemdMon::Notifiers
+  class Base
+    def initialize(options)
+      self.options = options
+      self.me      = self.class.name
+    end
+
+    # Subclasses must respond to a unit change
+    def notify!(notification)
+      raise "Notifier #{self.class} does not respond to notify!"
+    end
+
+    # Subclasses can choose to do something when SystemdMon starts
+    # E.g. with
+    #
+    # def notify_start!(hostname)
+    # end
+
+    # Subclasses can choose to do something when SystemdMon stops
+    # E.g. with
+    #
+    # def notify_stop!(hostname)
+    # end
+
+    def log(message)
+      SystemdMon::Logger.puts "#{me}: #{message}"
+    end
+
+    def debug(message = nil, &blk)
+      message = "#{me}: #{message}" if message
+      SystemdMon::Logger.debug message, &blk
+    end
+
+  protected
+    attr_accessor :options, :me
+  end
+end

data/lib/systemd_mon/notifiers/email.rb

@@ -0,0 +1,60 @@
+require 'mail'
+require 'systemd_mon/notifiers/base'
+require 'systemd_mon/logger'
+require 'systemd_mon/formatters/state_table_formatter'
+
+module SystemdMon::Notifiers
+  class Email < Base
+    def initialize(*)
+      super
+      if options['smtp']
+        opts = options
+        Mail.defaults do
+          delivery_method :smtp, Hash[opts['smtp'].map { |h, k| [h.to_sym, k] }]
+        end
+      end
+
+      validate_options!
+    end
+
+    def notify!(notification)
+      unit = notification.unit
+      subject = "#{unit.name} on #{notification.hostname}: #{unit.state_change.status_text}"
+      message = "Systemd unit #{unit.name} on #{notification.hostname} #{unit.state_change.status_text}: #{unit.state.active} (#{unit.state.sub})\n\n"
+      if unit.state_change.length > 1
+        message << SystemdMon::Formatters::StateTableFormatter.new(unit).as_text
+      end
+      message << "\nRegards, SystemdMon"
+
+      send_mail subject, message
+
+      log "sent email notification"
+    end
+
+  protected
+    attr_accessor :options
+
+    def validate_options!
+      unless options.has_key?("to")
+        raise NotifierError, "The 'to' address must be set to use the email notifier"
+      end
+      true
+    end
+
+    def send_mail(subject, message)
+      debug("Sending email to #{options['to']}:")
+      debug(%Q{ -> Subject: "#{subject}"})
+      debug(%Q{ -> Message: "#{message}"})
+
+      mail = Mail.new do
+        subject subject
+        body message
+      end
+      mail.to = options['to']
+      if options['from']
+        mail.from options['from']
+      end
+      mail.deliver!
+    end
+  end
+end

data/lib/systemd_mon/notifiers/slack.rb

@@ -0,0 +1,92 @@
+require 'slack-notifier'
+require 'systemd_mon/notifiers/base'
+
+module SystemdMon::Notifiers
+  class Slack < Base
+    def initialize(*)
+      super
+      self.notifier = ::Slack::Notifier.new(
+        options.fetch('team'),
+        options.fetch('token'),
+        channel: options['channel'],
+        username: options['username'],
+        icon_emoji: options['icon_emoji'],
+        icon_url: options['icon_url'])
+    end
+
+    def notify_start!(hostname)
+      message = "Startup notification for SystemdMon"
+
+      attach = {
+        fallback: message,
+        text: "SystemdMon is starting on #{hostname}",
+        color: "good"
+      }
+
+      notifier.ping message, attachments: [attach]
+    end
+
+    def notify_stop!(hostname)
+      message = "Shutdown alert for SystemdMon"
+
+      attach = {
+        fallback: message,
+        text: "SystemdMon is stopping on #{hostname}",
+        color: "danger"
+      }
+
+      notifier.ping message, attachments: [attach]
+    end
+
+    def notify!(notification)
+      unit = notification.unit
+      message = "Systemd unit #{unit.name} on #{notification.hostname} #{unit.state_change.status_text}"
+
+      attach = {
+        fallback: "#{message}: #{unit.state.active} (#{unit.state.sub})",
+        color: color(notification.type),
+        fields: fields(notification)
+      }
+
+      debug("sending slack message with attachment: ")
+      debug(attach.inspect)
+
+      notifier.ping message, attachments: [attach]
+      log "sent slack notification"
+    end
+
+  protected
+    attr_accessor :notifier
+
+    def fields(notification)
+      f = [
+        {
+          title: "Hostname",
+          value: notification.hostname,
+          short: true
+        },
+        {
+          title: "Unit",
+          value: notification.unit.name,
+          short: true
+        }
+      ]
+
+      changes = notification.unit.state_change.diff.map(&:last)
+      f.concat(changes.map { |v|
+        { title: v.display_name, value: v.value, short: true }
+      })
+    end
+
+    def color(type)
+      case type
+      when :alert
+        'danger'
+      when :info
+        '#0099CC'
+      else
+        'good'
+      end
+    end
+  end
+end

data/lib/systemd_mon/state.rb

@@ -0,0 +1,33 @@
+require 'systemd_mon/state_value'
+
+module SystemdMon
+  class State
+    include Enumerable
+
+    attr_reader :active, :sub, :loaded, :unit_file, :all_states
+
+    def initialize(active, sub, loaded, unit_file)
+      timestamp   = Time.now
+      @active     = StateValue.new("active", active, timestamp, %w(active), %w(inactive))
+      @sub        = StateValue.new("status", sub, timestamp)
+      @loaded     = StateValue.new("loaded", loaded, timestamp, %w(loaded))
+      @unit_file  = StateValue.new("file", unit_file, timestamp, %w(enabled), %w(disabled))
+      @all_states = [@active, @sub, @loaded, @unit_file]
+    end
+
+    def each
+      @all_states.each do |state|
+        yield state
+      end
+    end
+
+    def ok?
+      all?(&:ok?)
+    end
+
+    def fail?
+      any?(&:fail?)
+    end
+  end
+
+end

data/lib/systemd_mon/state_change.rb

@@ -0,0 +1,113 @@
+module SystemdMon
+  class StateChange
+    include Enumerable
+
+    attr_reader :states
+
+    def initialize(original_state = nil)
+      self.states = []
+      states << original_state if original_state
+    end
+
+    def last
+      states.last
+    end
+
+    def length
+      states.length
+    end
+
+    def <<(state)
+      self.states << state
+      @diff = nil
+    end
+
+    def each
+      states.each do |state|
+        yield state
+      end
+    end
+
+    def changes
+      states[1..-1]
+    end
+
+    def recovery?
+      first.fail? && last.ok?
+    end
+
+    def ok?
+      last.ok?
+    end
+
+    def fail?
+      last.fail?
+    end
+
+    def restart?
+      first.ok? && last.ok? && changes.any? { |s| s.active == "deactivating" }
+    end
+
+    def reload?
+      first.ok? && last.ok? && changes.any? { |s| s.active == "reloading" }
+    end
+
+    def still_fail?
+      length > 1 && first.fail? && last.fail?
+    end
+
+    def status_text
+      if recovery?
+        "recovered"
+      elsif restart?
+        "restarted"
+      elsif reload?
+        "reloaded"
+      elsif still_fail?
+        "still failed"
+      elsif fail?
+        "failed"
+      else
+        "started"
+      end
+    end
+
+    def important?
+      if length == 1
+        first.fail?
+      else
+        diff.map(&:last).any?(&:important?)
+      end
+    end
+
+    def diff
+      @diff ||= zipped.reject { |states|
+        match = states.first.value
+        states.all? { |s| s.value == match }
+      }
+    end
+
+    def zipped
+      if length == 1
+        first.all_states
+      else
+        first.all_states.zip(*changes.map(&:all_states))
+      end
+    end
+
+    def to_s
+      diff.inject("") { |s, (*states)|
+        first = states.shift
+        s << "#{first.name} state changed from #{first.value} to "
+        s << states.map(&:value).join(" then ")
+        s << "\n"
+        s
+      }
+    end
+
+  protected
+    attr_accessor :original, :changed
+    attr_writer :states
+  end
+
+end

data/lib/systemd_mon/state_value.rb

@@ -0,0 +1,48 @@
+module SystemdMon
+  class StateValue
+    attr_reader :name, :value, :ok_states, :failure_states, :timestamp
+
+    def initialize(name, value, timestamp, ok_states = [], failure_states = [])
+      self.name = name
+      self.value = value
+      self.ok_states = ok_states
+      self.failure_states = failure_states
+      self.timestamp = timestamp
+    end
+
+    def display_name
+      name.capitalize
+    end
+
+    def important?
+      ok_states.include?(value) || failure_states.include?(value)
+    end
+
+    def ok?
+      if ok_states.any?
+        ok_states.include?(value)
+      else
+        true
+      end
+    end
+
+    def fail?
+      if failure_states.any?
+        failure_states.include?(value)
+      else
+        false
+      end
+    end
+
+    def to_s
+      value
+    end
+
+    def ==(other)
+      value == other
+    end
+
+  protected
+    attr_writer :name, :value, :ok_states, :failure_states, :timestamp
+  end
+end

data/lib/systemd_mon/unit_with_state.rb

@@ -0,0 +1,33 @@
+require 'systemd_mon/state_change'
+
+module SystemdMon
+  class UnitWithState
+    attr_reader :unit, :state_change
+
+    def initialize(unit)
+      self.unit         = unit
+      self.state_change = StateChange.new
+    end
+
+    def name
+      unit.name
+    end
+
+    def <<(state)
+      self.state_change << state
+    end
+
+    def current_state
+      state_change.last
+    end
+
+    def reset!
+      self.state_change = StateChange.new(current_state)
+    end
+
+    alias :state :current_state
+
+  protected
+    attr_writer :state_change, :unit
+  end
+end

data/lib/systemd_mon/version.rb

@@ -0,0 +1,3 @@
+module SystemdMon
+  VERSION = "0.0.1"
+end

data/systemd_mon.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'systemd_mon/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "systemd_mon"
+  spec.version       = SystemdMon::VERSION
+  spec.authors       = ["Jon Cairns"]
+  spec.email         = ["jon@joncairns.com"]
+  spec.summary       = %q{Monitor systemd units and trigger alerts for failed states}
+  spec.description   = %q{Monitor systemd units and trigger alerts for failed states}
+  spec.homepage      = "https://github.com/joonty/systemd_mon"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "ruby-dbus"
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: systemd_mon
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jon Cairns
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby-dbus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Monitor systemd units and trigger alerts for failed states
+email:
+- jon@joncairns.com
+executables:
+- systemd_mon
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/systemd_mon
+- lib/systemd_mon.rb
+- lib/systemd_mon/callback_manager.rb
+- lib/systemd_mon/cli.rb
+- lib/systemd_mon/dbus_manager.rb
+- lib/systemd_mon/dbus_unit.rb
+- lib/systemd_mon/error.rb
+- lib/systemd_mon/formatters/base.rb
+- lib/systemd_mon/formatters/state_table_formatter.rb
+- lib/systemd_mon/logger.rb
+- lib/systemd_mon/monitor.rb
+- lib/systemd_mon/notification.rb
+- lib/systemd_mon/notification_centre.rb
+- lib/systemd_mon/notifier_loader.rb
+- lib/systemd_mon/notifiers/base.rb
+- lib/systemd_mon/notifiers/email.rb
+- lib/systemd_mon/notifiers/slack.rb
+- lib/systemd_mon/state.rb
+- lib/systemd_mon/state_change.rb
+- lib/systemd_mon/state_value.rb
+- lib/systemd_mon/unit_with_state.rb
+- lib/systemd_mon/version.rb
+- systemd_mon.gemspec
+homepage: https://github.com/joonty/systemd_mon
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Monitor systemd units and trigger alerts for failed states
+test_files: []