deadman_check 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 779989542c12fe1ed8b686a4386d2395c4f47258
-  data.tar.gz: 4d0a632f0aaefecca13b0b7050db36304917f4a9
+  metadata.gz: 3d86166c15a2c87baa6772b460cab729a6a7c341
+  data.tar.gz: 14e8ad5b0a9d9742f75aefe8eb01facadcd3d402
 SHA512:
-  metadata.gz: 57dd4ba651dc8533e16fa1064578a645edeba51f96166bdc15e75eb98274829725df894907f0259425a9c86871a77632879bca3b4747d2199deaff248c2df1ee
-  data.tar.gz: 5b5eab14128fbffb29bb7692e7483771f342eef5b58d656cf027b0545365a1fa0f7ce7a5fa0597905109a17101761ba734d520d2b99031f879e76f90f4eb1f3c
+  metadata.gz: 4aae7f90a0915a393a7aa8e6113af0862debe39a8498a1587869071636050f03b6eb0f09d5de2f287178f14547a1aa4efd153f90a77b0304b3fe6658f9140968
+  data.tar.gz: cff5f26ced3b63b1a2b4013953d0628d48e22346d051ffd9d0b5f214664fe63be6b5f39e7f0c8a98b017bc3dc601f29b4a905f50198e3bb2813a4244069233b8

data/.gitignore

@@ -8,3 +8,5 @@
 /spec/reports/
 /tmp/
 *.gem
+.slackapi
+

data/Dockerfile

@@ -0,0 +1,29 @@
+FROM ruby:2.4
+MAINTAINER Zane Williamson <zane.williamson@gmail.com>
+
+# Install apt packages
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -qq && \
+    apt-get install -y -qq \
+        less \
+        locales && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Configure locale
+ARG LOCALE="C.UTF-8"
+RUN locale-gen "$LOCALE" && \
+    dpkg-reconfigure locales
+ENV LANG="$LOCALE" LC_ALL="$LOCALE"
+
+ADD . /app/
+ADD lib /app/lib
+
+VOLUME /app
+WORKDIR /app
+
+RUN gem install bundler && \
+    bundle install && \
+    rake install
+
+ENTRYPOINT ["deadman-check"]

data/README.md

@@ -1,21 +1,164 @@
 # DeadmanCheck
 
-[![Build Status](https://travis-ci.org/sepulworld/deadman_check.svg)](https://travis-ci.org/sepulworld/deadman_check)
-[![Gem Version](https://badge.fury.io/rb/deadman_check.svg)](http://badge.fury.io/rb/deadman_check)
+[![Build Status](https://travis-ci.org/sepulworld/deadman-check.svg)](https://travis-ci.org/sepulworld/deadman_check)
+[![Gem Version](https://badge.fury.io/rb/deadman-check.svg)](http://badge.fury.io/rb/deadman_check)
+
+A monitoring companion for Nomad periodic jobs that alerts if periodic jobs are
+not processing as expected. The deadman-check has 2 modes, one to run with the
+Nomad periodic job as an additional [task](https://www.nomadproject.io/docs/job-specification/task.html) to update a key in Redis with current EPOCH time. The other mode is of deadman-check
+is intended to run as a separate process that will monitor the Redis key's EPOCH
+time value and alert if that value fails to meet a time 'freshness' threshold that
+is expected for that job.
+
+* requires a Redis instance
+* alerting requires a SLACK_API_TOKEN environment variable
+
+## Example Usage
+
+Let's say I have a Nomad periodic job that is set to run every 10 minutes. The Nomad configuration looks like this:
+
+```hcl
+job "SilverBulletPeriodic" {
+  type = "batch"
+
+  periodic {
+    cron             = "*/10 * * * * *"
+    prohibit_overlap = true
+  }
+
+  group "utility" {
+    task "SilverBulletPeriodicProcess" {
+      driver = "docker"
+      config {
+        image    = "silverbullet:build_1"
+        work_dir = "/utility/silverbullet"
+        command  = "blaster"
+      }
+      resources {
+        cpu = 100
+        memory = 500
+      }
+    }
+  }
+}
+```
 
-Monitor a Redis key that contains an EPOCH time entry. Send email if EPOCH age hits given threshold
+To monitor the SilverBulletPeriodicProcess task let's add a deadmad-check task to
+run post updates to a Redis endpoint (10.0.0.1 for this example)
+
+```hcl
+job "SilverBulletPeriodic" {
+  type = "batch"
+
+  periodic {
+    cron             = "*/10 * * * * *"
+    prohibit_overlap = true
+  }
+
+  group "silverbullet" {
+    task "SilverBulletPeriodicProcess" {
+      driver = "docker"
+      config {
+        image    = "silverbullet:build_1"
+        work_dir = "/utility/silverbullet"
+        command  = "blaster"
+      }
+      resources {
+        cpu = 100
+        memory = 500
+      }
+    }
+    task "DeadmanSetSilverBulletPeriodicProcess" {
+      driver = "docker"
+      config {
+        image    = "sepulworld/deadman-check"
+        command  = "key_set"
+        args     = [
+          "--host",
+          "10.0.0.1",
+          "--port",
+          "6379",
+          "--key",
+          "deadman/SilverBulletPeriodicProcess"]
+      }
+      resources {
+        cpu = 100
+        memory = 256
+      }
+    }
+  }
+}
+```
 
-## Installation
+Now the key, deadman/SilverBulletPeriodicProcess, in redis1.mydomain.net will be updated with
+the EPOCH time for each SilverBulletPeriodic job run. If the job hangs or fails to run
+we will know via the EPOCH time entry going stale.
+
+Next we need a job that will run to monitor this key.
+
+```hcl
+job "DeadmanMonitoring" {
+  type = "service"
+
+  group "monitor" {
+    task "DeadmanMonitorSilverBulletPeriodicProcess" {
+      driver = "docker"
+      config {
+        image    = "sepulworld/deadman-check"
+        command  = "switch_monitor"
+        args     = [
+          "--host",
+          "10.0.0.1",
+          "--port",
+          "6379",
+          "--key",
+          "deadman/SilverBulletPeriodicProcess",
+          "--freshness",
+          "800",
+          "--alert-to",
+          "#slackroom",
+          "--daemon",
+          "--daemon-sleep",
+          "900"]
+      }
+      resources {
+        cpu = 100
+        memory = 256
+      }
+      env {
+        SLACK_API_TOKEN = "YourSlackApiToken"
+      }
+    }
+  }
+}
+```
 
-And then execute:
+Monitor a Redis key that contains an EPOCH time entry. Send a Slack message if EPOCH age hits given threshold
 
-    $ bundle
+## Local system installation
 
-Or install it yourself as:
+execute:
 
+    $ bundle install
     $ gem install deadman_check
 
-## Usage
+## Install and run deadman-check from Docker
+
+```
+# Optional: If you don't pull explicitly, `docker run` will do it for you
+$ docker pull sepulworld/deadman-check
+
+$ alias deadman-check='\
+  docker run \
+    -it --rm --name=deadman-check \
+    sepulworld/deadman-check'
+```
+
+(Depending on how your system is set up, you might have to add sudo in front of the above docker commands or add your user to the docker group).
+
+If you don't do the docker pull, the first time you run deadman-check, the docker run command will automatically pull the sepulworld/deadman-check image on the Docker Hub. Subsequent runs will use a locally cached copy of the image and will not have to download anything.
+
+## Usage via Local System Install
 
 ```bash
 $ deadman-check -h
@@ -26,7 +169,7 @@ $ deadman-check -h
   DESCRIPTION:
 
     Monitor a Redis key that contains an EPOCH time entry.
-      Send email if EPOCH age hits given threshold
+      Send a Slack message if EPOCH age hits given threshold
 
   COMMANDS:
 
@@ -106,8 +249,7 @@ $ deadman-check switch_monitor -h
       --port 6379 \
       --key deadman/myservice \
       --freshness 500 \
-      --alert-to ops@mycomany.tld \
-      --alert-from ops-no-reply-email@mycomany.tld
+      --alert-to #slackroom
 
   OPTIONS:
 
@@ -124,11 +266,14 @@ $ deadman-check switch_monitor -h
         The value in seconds to alert on when the recorded
             EPOCH value exceeds current EPOCH
 
-    --alert-to EMAIL
-        Email address to send alert to
+    --alert-to SLACKROOM
+        Slackroom to alert to
+
+    --daemon
+        Run as a daemon, otherwise will run check just once
 
-    --alert-from EMAIL
-        Email address to send from
+    --daemon-sleep SECONDS
+        Set the number of seconds to sleep in between switch checks, default 300
 ```
 
 ## Development

data/Rakefile

@@ -8,3 +8,18 @@ Rake::TestTask.new(:test) do |t|
 end
 
 task :default => :test
+
+desc "Docker build image"
+task :docker_build do
+  sh %{docker build -t sepulworld/deadman-check .}
+end
+
+desc "Push Docker image to Docker Hub"
+task :docker_push do
+  sh %{docker push sepulworld/deadman-check}
+end
+
+desc "Pull Docker image to Docker Hub"
+task :docker_pull do
+  sh %{docker pull sepulworld/deadman-check}
+end

data/bin/deadman-check

@@ -3,30 +3,35 @@
 require 'rubygems'
 require 'commander/import'
 require 'deadman_check'
+require 'daemons'
 
 program :name, 'deadman-check'
 program :version, DeadmanCheck::VERSION
-program :description, %q{Monitor a Redis key that contains an EPOCH time entry.
-  Send email if EPOCH age hits given threshold}
+program :description, %q{Monitor a Redis key that contains an EPOCH time entry. Send email if EPOCH age hits given threshold}
 
 command :switch_monitor do |c|
   c.syntax = 'deadman-check switch_monitor [options]'
   c.summary = 'Target a Redis key to monitor'
   c.description = ''
-  c.example %q{Target a Redis key deadman/myservice, and this key has an EPOCH
-     value to check looking to alert on 500 second or greater freshness},
+  c.example %q{Target a Redis key deadman/myservice, and this key has an EPOCH value to check looking to alert on 500 second or greater freshness},
     %q{deadman-check switch_monitor --host 127.0.0.1 --port 6379 --key deadman/myservice --freshness 500 --alert-to ops@mycomany.tld --alert-from ops-no-reply-email@mycomany.tld}
   c.option '--host HOST', String, 'IP address or hostname of Redis system'
   c.option '--port PORT', String, 'port Redis is listening on'
   c.option '--key KEY', String, 'Redis key to monitor'
   c.option '--freshness SECONDS', String, %q{The value in seconds to alert on when the recorded
     EPOCH value exceeds current EPOCH}
-  c.option '--alert-to EMAIL', String, 'Email address to send alert to'
-  c.option '--alert-from EMAIL', String, 'Email address to send from'
+  c.option '--alert-to SLACKROOM', String, 'Slackroom to alert to'
+  c.option '--daemon', 'Run as a daemon, otherwise will run check just once'
+  c.option '--daemon-sleep SECONDS', String, 'Set the number of seconds to sleep in between switch checks, default 300'
   c.action do |args, options|
+    options.default :daemon_sleep => 300
     switch_monitor = DeadmanCheck::SwitchMonitor.new(options.host, options.port,
-      options.key, options.freshness, options.alert_to, options.alert_from)
-    switch_monitor.run_check
+      options.key, options.freshness, options.alert_to, options.daemon_sleep)
+    if options.daemon
+      Daemons.run(switch_monitor.run_check_daemon)
+    else
+      switch_monitor.run_check_once
+    end
   end
 end
 
@@ -38,7 +43,7 @@ command :key_set do |c|
     %q{deadman-check key_set --host 127.0.0.1 --port 6379 --key deadman/myservice}
   c.option '--host HOST', String, 'IP address or hostname of Redis system'
   c.option '--port PORT', String, 'port Redis is listening on'
-  c.option '--key KEY', String, 'Redis key to monitor'
+  c.option '--key KEY', String, 'Redis key to report EPOCH time to'
   c.action do |args, options|
     key_set = DeadmanCheck::KeySet.new(options.host, options.port,
       options.key)

data/deadman_check.gemspec

@@ -36,5 +36,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'commander', '~> 4.4', '>= 4.4.3'
   spec.add_dependency 'redis', '~> 3.3', '>= 3.3.3'
-  spec.add_dependency 'pony', '~> 1.1'
+  spec.add_dependency 'slack-ruby-client', '~> 0.8.0'
+  spec.add_dependency 'daemons', '~> 1.2.4', '>=1.2.4'
 end

data/lib/deadman_check/version.rb

@@ -1,3 +1,3 @@
 module DeadmanCheck
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/lib/deadman_check_switch.rb

@@ -1,20 +1,24 @@
 require 'deadman_check/version'
 require 'deadman_check_global'
 require 'redis'
-require 'pony'
+require 'slack-ruby-client'
 
 module DeadmanCheck
   # Switch class
   class SwitchMonitor
-    attr_accessor :host, :port, :key, :freshness, :alert_to, :alert_from
+    attr_accessor :host, :port, :key, :freshness, :alert_to, :daemon_sleep
 
-    def initialize(host, port, key, freshness, alert_to, alert_from)
+    def initialize(host, port, key, freshness, alert_to, daemon_sleep)
       @host = host
       @port = port
       @key  = key
       @freshness = freshness.to_i
       @alert_to = alert_to
-      @alert_from = alert_from
+      @daemon_sleep = daemon_sleep.to_i
+    end
+
+    Slack.configure do |config|
+      config.token = ENV['SLACK_API_TOKEN']
     end
 
     def _diff_epoc(current_epoch, recorded_epoch)
@@ -28,21 +32,26 @@ module DeadmanCheck
       return recorded_epoch
     end
 
-    def email_alert(alert_to, alert_from, key, recorded_epoch, current_epoch,
-      epoch_diff)
-      Pony.mail(:to => alert_to, :from => alert_from,
-        :subject => "Alert: Deadman Switch Triggered for #{key}",
-        :body => "Alert: Deadman Switch Triggered for #{key}, with
-          #{epoch_diff} seconds since last run")
+    def slack_alert(alert_to, key, epoch_diff)
+      client = Slack::Web::Client.new
+      client.chat_postMessage(channel: alert_to, text: "Alert: Deadman Switch
+        Triggered for #{key}, with #{epoch_diff} seconds since last run",
+        username: 'deadman')
     end
 
-    def run_check
+    def run_check_once
       recorded_epoch = _get_recorded_epoch(@host, @port, @key).to_i
       current_epoch = DeadmanCheck::DeadmanCheckGlobal.new.get_epoch_time.to_i
       epoch_diff = _diff_epoc(current_epoch, recorded_epoch)
       if epoch_diff > @freshness
-        email_alert(@alert_to, @alert_from, @key,
-        recorded_epoch, current_epoch, epoch_diff)
+        slack_alert(@alert_to, @key, epoch_diff)
+      end
+    end
+
+    def run_check_daemon
+      loop do
+        run_check_once
+        sleep(@daemon_sleep)
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deadman_check
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - zane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-21 00:00:00.000000000 Z
+date: 2017-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -93,19 +93,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.3.3
 - !ruby/object:Gem::Dependency
-  name: pony
+  name: slack-ruby-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.4
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.4
 description: |-
   A script to check a given Redis key EPOCH for
       freshness. Good for monitoring cron jobs or batch jobs. Have the last step
@@ -121,6 +141,7 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
+- Dockerfile
 - Gemfile
 - LICENSE.txt
 - README.md