systemd_mon_mod 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 06a413928718c467090adf686c9dfa4d1f869077389f0fe83a81889066e87ef9
+  data.tar.gz: d7b6dd5c3f69ccd71e650ff4cbb5d99421f1344b1f6cb8f46fe2bf8830cb2257
+SHA512:
+  metadata.gz: 4338ab309e663836cbe70a0958ee1fbc682f63e8f6c2406a18f56fc8d71d026e9c5928c87a8b82341bdc2f40a9b676f124478767aa84b8a958c5a31266526715
+  data.tar.gz: a441af6f624aa118b059288743a3b1bb8cee6fb79e1fb99aabdf67ae0973261642a1e9f788757e0eaa3963c12c65ca2c7273d97ec7ca4c4c9cb8fcfeeae058d3

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+.vagrant

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in systemd_alert.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Jon Cairns
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,125 @@
+# SystemdMon
+
+Monitor systemd units and trigger alerts for failed states. The command line tool runs as a daemon, using dbus to get notifications of changes to systemd services. If a service enters a failed state, or returns from a failed state to an active state, notifications will be triggered.
+
+Built-in notifications include email, slack, and hipchat, but more can be added via the ruby API.
+
+It works by subscribing to DBus notifications from Systemd. This means that there is no polling, and no busy-loops. SystemdMon will sit in the background, happily waiting and using minimal processes.
+
+## Requirements
+
+* A linux server
+* Ruby > 1.9.3
+* Systemd (v204 was used in development)
+* `mail` gem (if email notifier is used)
+* `slack-notifier` gem > 1.0 (if slack notifier is used)
+* `hipchat` (if hipchat notifier is used)
+
+## Installation
+
+Install the gem using:
+
+    gem install systemd_mon
+
+## Usage
+
+To run the command line tool, you will first need to create a YAML configuration file to specify which systemd units you want to monitor, and which notifications you want to trigger. A full example looks like this:
+
+```yaml
+---
+verbose: true # Default is off
+notifiers:
+  email:
+    to: "team@mydomain.com"
+    from: "systemdmon@mydomain.com"
+    # These are options passed to the 'mail' gem
+    smtp:
+        address: smtp.gmail.com
+        port: 587
+        domain: mydomain.com
+        user_name: "user@mydomain.com"
+        password: "supersecr3t"
+        authentication: "plain"
+        enable_starttls_auto: true
+  slack:
+    webhook_url: https://hooks.slack.com/services/super/secret/tokenthings
+    channel: mychannel
+    username: doge
+    icon_emoji: ":computer"
+    icon_url: "http://example.com/icon"
+  hipchat:
+    token: bigsecrettokenhere
+    room: myroom
+    username: doge
+units:
+- unicorn.service
+- nginx.service
+- sidekiq.service
+```
+
+Save that somewhere appropriate (e.g. `/etc/systemd_mon.yml`), then start the command line tool with:
+
+    $ systemd_mon /etc/systemd_mon.yml
+
+You'll probably want to run it via systemd, which you can do with this example service file (change file paths as appropriate):
+
+```
+[Unit]
+Description=SystemdMon
+After=network.target
+
+[Service]
+Type=simple
+User=deploy
+StandardInput=null
+StandardOutput=syslog
+StandardError=syslog
+ExecStart=/usr/local/bin/systemd_mon /etc/systemd_mon.yml
+
+[Install]
+WantedBy=multi-user.target
+```
+
+## Behaviour
+
+Systemd provides information about state changes in very fine detail. For example, if you start a service, it may go through the following states: activating (start-pre), activiating (start) and finally active (running). This will likely happen in less than a second, and you probably don't want 3 notifications. Therefore, SystemdMon queues up states until it comes across one that you think you should know about. In this case, it will notify you when the state reaches active (running), but the notification can show the history of how the state changed so you get the full picture.
+
+SystemdMon does simple analysis on the history of state changes, so it can summarise with statuses like "recovered", "automatically restarted", "still failed", etc. It will also report with the host name of the server.
+
+You'll also want to know if SystemdMon itself falls over, and when it starts back up again. It will attempt to send a final notification before it exits, and one to say it's starting. However, be aware that it might not send a notification in some conditions (e.g. in the case of a SIGKILL), or a network failure. The age-old question: who will watch the watcher?
+
+## Docker integration
+There is a public Docker image available which bundles all requirements (Ruby + Gems). Since systemd_mon relies on dbus, you need to mount the host dbus directory into your container. Besides that, the configuration filename is currently hardcoded to systemd_mon.yml. You have to mount the directory where the systemd_mon.yml file is located on your host system into your container as well. Below is a working example: 
+
+```
+docker run --name "systemd_mon" -v /var/run/dbus:/var/run/dbus -v /path/to/systemd_mon/config/:/systemd_mon/ kromit/systemd_mon
+```
+
+If you want to run this image with systemd (very handy on CoreOS for example) you can use it as follows:
+
+```
+[Unit]
+Description=systemd_mon
+After=docker.service
+Requires=docker.service
+
+[Service]
+Restart=always
+RestartSec=60
+ExecStartPre=-/usr/bin/docker kill systemd_mon
+ExecStartPre=-/usr/bin/docker rm systemd_mon
+ExecStart=/usr/bin/docker run --name "systemd_mon" -v /var/run/dbus:/var/run/dbus -v /path/to/systemd_mon/config/:/systemd_mon/ kromit/systemd_mon
+
+[Install]
+WantedBy=multi-user.target
+```
+
+## Contributing
+
+I'd love more contributions, particulary new notifiers. Follow the example of the slack and email notifiers and either package as a new gem or submit a pull request if you think it should be part of the main project.
+
+1. Fork it ( https://github.com/joonty/systemd_mon/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/systemd_mon

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'systemd_mon/cli'
+
+SystemdMon::CLI.new.start

data/lib/systemd_mon/callback_manager.rb

@@ -0,0 +1,39 @@
+require 'systemd_mon/unit_with_state'
+
+module SystemdMon
+  class CallbackManager
+    def initialize(queue)
+      self.queue  = queue
+      self.states = Hash.new { |h, u| h[u] = UnitWithState.new(u) }
+    end
+
+    def start(change_callback, each_state_change_callback)
+      loop do
+        unit, state = queue.deq
+        Logger.debug { "#{unit} new state: #{state}" }
+        unit_state = states[unit]
+        unit_state << state
+
+        if each_state_change_callback
+          with_error_handling { each_state_change_callback.call(unit_state) }
+        end
+
+        if change_callback && unit_state.state_change.important?
+          with_error_handling { change_callback.call(unit_state) }
+        end
+
+        unit_state.reset! if unit_state.state_change.important?
+      end
+    end
+
+    def with_error_handling
+      yield
+    rescue => e
+      Logger.error "Uncaught exception (#{e.class}) in callback: #{e.message}"
+      Logger.debug_error { "\n\t#{e.backtrace.join("\n\t")}\n" }
+    end
+
+  protected
+    attr_accessor :queue, :states
+  end
+end

data/lib/systemd_mon/cli.rb

@@ -0,0 +1,86 @@
+require 'yaml'
+require 'systemd_mon'
+require 'systemd_mon/monitor'
+require 'systemd_mon/error'
+require 'systemd_mon/dbus_manager'
+
+module SystemdMon
+  class CLI
+    def initialize
+      self.me      = "systemd_mon"
+      self.verbose = true
+    end
+
+    def start
+      yaml_config_file = ARGV.first
+      self.options = load_and_validate_options(yaml_config_file)
+      self.verbose = options['verbose'] || false
+      Logger.verbose = verbose
+
+      start_monitor
+
+    rescue SystemdMon::Error => e
+      err_string = e.message
+      if verbose
+        if e.original
+          err_string << " - #{e.original.message} (#{e.original.class})"
+          err_string << "\n\t#{e.original.backtrace.join("\n\t")}"
+        else
+          err_string << " (#{e.class})"
+          err_string << "\n\t#{e.backtrace.join("\n\t")}"
+        end
+      end
+      fatal_error(err_string)
+    rescue => e
+      err_string = e.message
+      if verbose
+        err_string << " (#{e.class})"
+        err_string << "\n\t#{e.backtrace.join("\n\t")}"
+      end
+      fatal_error(err_string)
+    end
+
+  protected
+    def start_monitor
+      monitor = Monitor.new(DBusManager.new)
+
+      # Load units to monitor
+      monitor.register_units options['units']
+
+      options['notifiers'].each do |name, notifier_options|
+        klass = NotifierLoader.new.get_class(name)
+        monitor.add_notifier klass.new(notifier_options)
+      end
+
+      monitor.start
+    end
+
+    def load_and_validate_options(yaml_config_file)
+      options = load_options(yaml_config_file)
+
+      unless options.has_key?('notifiers') && options['notifiers'].any?
+        fatal_error("no notifiers have been defined, there is no reason to continue")
+      end
+      unless options.has_key?('units') && options['units'].any?
+        fatal_error("no units have been added for watching, there is no reason to continue")
+      end
+      options
+    end
+
+    def load_options(yaml_config_file)
+      unless yaml_config_file && File.exists?(yaml_config_file)
+        fatal_error "First argument must be a path to a YAML configuration file"
+      end
+
+      YAML.load_file(yaml_config_file)
+    end
+
+    def fatal_error(message, code = 255)
+      $stderr.puts " #{me} error: #{message}"
+      exit code
+    end
+
+  protected
+    attr_accessor :verbose, :options, :me
+  end
+end

data/lib/systemd_mon/dbus_manager.rb

@@ -0,0 +1,35 @@
+require 'dbus'
+require 'systemd_mon/error'
+require 'systemd_mon/dbus_unit'
+
+module SystemdMon
+  class DBusManager
+    def initialize
+      self.dbus            = DBus::SystemBus.instance
+      self.systemd_service = dbus.service("org.freedesktop.systemd1")
+      self.systemd_object  = systemd_service.object("/org/freedesktop/systemd1")
+      systemd_object.introspect
+      if systemd_object.respond_to?("Subscribe")
+        systemd_object.Subscribe
+      else
+        raise SystemdMon::SystemdError, "Systemd is not installed, or is an incompatible version. It must provide the Subscribe dbus method: version 204 is the minimum recommended version."
+      end
+    end
+
+    def fetch_unit(unit_name)
+      path = systemd_object.GetUnit(unit_name).first
+      DBusUnit.new(unit_name, path, systemd_service.object(path))
+    rescue DBus::Error
+      raise SystemdMon::UnknownUnitError, "Unknown or unloaded systemd unit '#{unit_name}'"
+    end
+
+    def runner
+      main = DBus::Main.new
+      main << dbus
+      main
+    end
+
+  protected
+    attr_accessor :systemd_service, :systemd_object, :dbus
+  end
+end

data/lib/systemd_mon/dbus_unit.rb

@@ -0,0 +1,70 @@
+require 'systemd_mon/state'
+
+module SystemdMon
+  class DBusUnit
+    attr_reader :name, :maybe_service_type
+
+    IFACE_UNIT    = "org.freedesktop.systemd1.Unit"
+    IFACE_SERVICE = "org.freedesktop.systemd1.Service"
+    IFACE_PROPS   = "org.freedesktop.DBus.Properties"
+
+    def initialize(name, path, dbus_object)
+      self.name = name
+      self.path = path
+      self.dbus_object = dbus_object
+      prepare_dbus_objects!
+      self.maybe_service_type = service_type
+    end
+
+    def register_listener!(queue)
+      queue.enq [self, build_state] # initial state
+      dbus_object.on_signal("PropertiesChanged") do |iface|
+        if iface == IFACE_UNIT
+          queue.enq [self, build_state]
+        end
+      end
+    end
+
+    def on_change(&callback)
+      self.change_callback = callback
+    end
+
+    def on_each_state_change(&callback)
+      self.each_state_change_callback = callback
+    end
+
+    def property(name)
+      dbus_object.Get(IFACE_UNIT, name).first
+    end
+
+    def to_s
+      "#{name}" << (maybe_service_type ? " (#{maybe_service_type})" : '')
+    end
+
+  protected
+    attr_accessor :path, :dbus_object, :change_callback, :each_state_change_callback
+    attr_writer   :name, :maybe_service_type
+
+    def build_state
+      State.new(
+        property("ActiveState"),
+        property("SubState"),
+        property("LoadState"),
+        property("UnitFileState"),
+        maybe_service_type
+      )
+    end
+
+    def prepare_dbus_objects!
+      dbus_object.introspect
+      self.dbus_object.default_iface = IFACE_PROPS
+      self
+    end
+
+    def service_type
+      if dbus_object[IFACE_SERVICE]
+        dbus_object[IFACE_SERVICE]['Type']
+      end
+    end
+  end
+end

data/lib/systemd_mon/error.rb

@@ -0,0 +1,19 @@
+module SystemdMon
+
+  # Save original exception for use in verbose mode
+  class Error < StandardError
+    attr_reader :original
+
+    def initialize(msg, original=$!)
+      super(msg)
+      @original = original
+    end
+  end
+
+  class SystemdError < Error; end
+  class MonitorError < Error; end
+  class UnknownUnitError < Error; end
+  class NotificationError < Error; end
+  class NotifierDependencyError < Error; end
+  class NotifierError < Error; end
+end

data/lib/systemd_mon/formatters/base.rb

@@ -0,0 +1,18 @@
+module SystemdMon::Formatters
+  class Base
+    def initialize(unit)
+      self.unit = unit
+    end
+
+    def as_html
+      raise "The formatter #{self.class} does not provide an html formatted string"
+    end
+
+    def as_text
+      raise "The formatter #{self.class} does not provide a plain text string"
+    end
+
+  protected
+    attr_accessor :unit
+  end
+end

data/lib/systemd_mon/formatters/state_table_formatter.rb

@@ -0,0 +1,32 @@
+require 'systemd_mon/formatters/base'
+module SystemdMon::Formatters
+  class StateTableFormatter < Base
+    def as_text
+      table = render_table
+      lengths = table.transpose.map { |v| v.map(&:length).max }
+
+      full_width = lengths.inject(&:+) + (lengths.length * 3) + 1
+      div = " " + ("-" * full_width) + "\n"
+      s = div.dup
+      table.each do |row|
+        s << " | "
+        row.each_with_index { |col, i|
+          s << col.ljust(lengths[i]) + " | "
+        }
+        s << "\n" + div.dup
+      end
+      s
+    end
+
+  protected
+    def render_table
+      changed = unit.state_change.diff
+      table = []
+      table << ["Time"].concat(changed.map{|v| v.first.display_name})
+      changed.transpose.each do |vals|
+        table << [vals.first.timestamp.strftime("%H:%M:%S.%3N %z")].concat(vals.map{|v| v.value})
+      end
+      table
+    end
+  end
+end

data/lib/systemd_mon/logger.rb

@@ -0,0 +1,33 @@
+module SystemdMon
+  class Logger
+    def self.verbose=(flag)
+      @verbose = flag
+    end
+
+    def self.verbose
+      @verbose
+    end
+
+    def self.debug(message = nil, stream = $stdout)
+      if verbose
+        if block_given?
+          $stdout.puts yield
+        else
+          $stdout.puts message
+        end
+      end
+    end
+
+    def self.error(message = nil)
+      $stderr.puts message
+    end
+
+    def self.debug_error(message = nil)
+      debug message, $stderr
+    end
+
+    def self.puts(message = nil)
+      $stdout.puts message
+    end
+  end
+end

data/lib/systemd_mon/monitor.rb

@@ -0,0 +1,98 @@
+require 'thread'
+require 'systemd_mon/logger'
+require 'systemd_mon/callback_manager'
+require 'systemd_mon/notification_centre'
+require 'systemd_mon/notification'
+require 'systemd_mon/error'
+
+module SystemdMon
+  class Monitor
+    def initialize(dbus_manager)
+      self.hostname     = `hostname`.strip
+      self.dbus_manager = dbus_manager
+      self.units        = []
+      self.change_callback     = lambda(&method(:unit_change_callback))
+      self.notification_centre = NotificationCentre.new
+      Thread.abort_on_exception = true
+    end
+
+    def add_notifier(notifier)
+      notification_centre << notifier
+      self
+    end
+
+    def register_unit(unit_name)
+      self.units << dbus_manager.fetch_unit(unit_name)
+      self
+    end
+
+    def register_units(*unit_names)
+      self.units.concat unit_names.flatten.map { |unit_name|
+        dbus_manager.fetch_unit(unit_name)
+      }
+      self
+    end
+
+    def on_change(&callback)
+      self.change_callback = callback
+      self
+    end
+
+    def on_each_state_change(&callback)
+      self.each_state_change_callback = callback
+      self
+    end
+
+    def start
+      startup_check!
+      at_exit { notification_centre.notify_stop! hostname }
+      notification_centre.notify_start! hostname
+
+      Logger.puts "Monitoring changes to #{units.count} units"
+      Logger.debug { " - " + units.map(&:name).join("\n - ") + "\n\n" }
+      Logger.debug { "Using notifiers: #{notification_centre.classes.join(", ")}"}
+
+      state_q = Queue.new
+
+      units.each do |unit|
+        unit.register_listener! state_q
+      end
+
+      [start_callback_thread(state_q),
+       start_dbus_thread].each(&:join)
+    end
+
+protected
+    attr_accessor :units, :dbus_manager, :change_callback, :each_state_change_callback, :hostname, :notification_centre
+
+    def startup_check!
+      unless units.any?
+        raise MonitorError, "At least one systemd unit should be registered before monitoring can start"
+      end
+      unless notification_centre.any?
+        raise MonitorError, "At least one notifier should be registered before monitoring can start"
+      end
+      self
+    end
+
+    def start_dbus_thread
+      Thread.new do
+        dbus_manager.runner.run
+      end
+    end
+
+    def start_callback_thread(state_q)
+      Thread.new do
+        manager = CallbackManager.new(state_q)
+        manager.start change_callback, each_state_change_callback
+      end
+    end
+
+    def unit_change_callback(unit)
+      Logger.puts "#{unit.name} #{unit.state_change.status_text}: #{unit.state.active} (#{unit.state.sub})"
+      Logger.debug unit.state_change.to_s
+      Logger.puts
+      notification_centre.notify! Notification.new(hostname, unit)
+    end
+  end
+end

data/lib/systemd_mon/notification.rb

@@ -0,0 +1,34 @@
+module SystemdMon
+  class Notification
+    attr_reader :unit, :type, :hostname
+
+    def initialize(hostname, unit)
+      self.hostname = hostname
+      self.unit     = unit
+      self.type     = determine_type
+    end
+
+    def self.types
+      [:alert, :warning, :info, :ok]
+    end
+
+    def type_text
+      type.to_s.capitalize
+    end
+
+  protected
+    attr_writer :unit, :type, :hostname
+
+    def determine_type
+      if unit.state_change.ok?
+        if unit.state_change.first.fail?
+          :ok
+        else
+          :info
+        end
+      else
+        :alert
+      end
+    end
+  end
+end