deadman_check 0.1.11 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12920b0224825d46cdd41699ae4221770170d1de
-  data.tar.gz: 4979ff88b6ac486c39a7fe3b8a00ace5e2567b45
+  metadata.gz: 60016a96e65047af901a74875d61fd0052cebf86
+  data.tar.gz: 08e97ecbae9e2d7cc68dc80b1dbc0153700ff9f8
 SHA512:
-  metadata.gz: def14d2515337b1adf18abdce2f460241eb5a33a21bd2e33103cdbc2e4778644e4fbb816e12155f19ff1454b70050a5200bdd69fed857f7289a6ac38b881e1fb
-  data.tar.gz: 9e2eb7f0f3113d4b8b6b5c11f1d21c2e8fa43516d556820b2cc76014ea444db480ea73662501ce2874eb2999bc436787f14132e371fd90dc26013ecb62fc433a
+  metadata.gz: 8e41ca251ccfa94ad4731b19a8b9f4ed61fbbecaae6dd9eb4d94517e15faeb640adbb867a2c7c24ef18357081718e467b2096d6406929513bddce67db1b9e129
+  data.tar.gz: c9b409c2cf717875c3645567dba926bc0a57ee659d6e3daebf9cfec1b896342267b8e7870ebd27dee38986ef1411f17b550d069c927f8763bad12311c81371d0

data/README.md

@@ -89,7 +89,9 @@ job "SilverBulletPeriodic" {
           "--port",
           "8500",
           "--key",
-          "deadman/SilverBulletPeriodicProcess"]
+          "deadman/SilverBulletPeriodicProcess",
+          "--frequency",
+          "700"]
       }
       resources {
         cpu = 100
@@ -99,8 +101,7 @@ job "SilverBulletPeriodic" {
   }
 }
 ```
-
-<img width="1215" alt="screen shot 2017-03-26 at 3 28 50 pm" src="https://cloud.githubusercontent.com/assets/538171/24335809/276147ec-1239-11e7-83d8-0fca95bebdc2.png">
+<img width="1215" alt="screen shot 2017-04-23 at 11 14 36 pm" src="https://cloud.githubusercontent.com/assets/538171/25324439/b65541d6-287a-11e7-9b6d-4e1c9565eed2.png">
 
 Now the key, deadman/SilverBulletPeriodicProcess, at 10.0.0.1 will be updated with
 the EPOCH time for each SilverBulletPeriodic job run. If the job hangs or fails to run
@@ -126,8 +127,6 @@ job "DeadmanMonitoring" {
           "8500",
           "--key",
           "deadman/SilverBulletPeriodicProcess",
-          "--freshness",
-          "800",
           "--alert-to",
           "slackroom",
           "--daemon",
@@ -146,10 +145,52 @@ job "DeadmanMonitoring" {
 }
 ```
 
-Monitor a Consul key that contains an EPOCH time entry. Send a Slack message if EPOCH age hits given threshold
+Monitor a Consul key that contains an EPOCH time entry. Send a Slack message if EPOCH age hits given frequency threshold
 
 <img width="752" alt="screen shot 2017-03-26 at 3 29 28 pm" src="https://cloud.githubusercontent.com/assets/538171/24335811/2e57eee8-1239-11e7-9fff-c8a10d956f2e.png">
 
+If you have multiple periodic jobs that need to be monitored then use the ```--key-path``` argument instead of ```--key```. Be sure to ```key_set``` all under the same Consul key path.
+
+<img width="658" alt="screen shot 2017-04-23 at 11 17 29 pm" src="https://cloud.githubusercontent.com/assets/538171/25324510/14d6e7f0-287b-11e7-9c0d-733d69e1cc94.png">
+
+To monitor the above you would just use the ```--key-path``` argument instead of ```--key```
+
+```hcl
+job "DeadmanMonitoring" {
+  datacenters = ["dc1"]
+  type = "service"
+
+  group "monitor" {
+    task "DeadmanMonitorSilverBulletPeriodicProcesses" {
+      driver = "docker"
+      config {
+        image    = "sepulworld/deadman-check"
+        command  = "switch_monitor"
+        args     = [
+          "--host",
+          "10.0.0.1",
+          "--port",
+          "8500",
+          "--key-path",
+          "deadman/",
+          "--alert-to",
+          "slackroom",
+          "--daemon",
+          "--daemon-sleep",
+          "900"]
+      }
+      resources {
+        cpu = 100
+        memory = 256
+      }
+      env {
+        SLACK_API_TOKEN = "YourSlackApiToken"
+      }
+    }
+  }
+}
+```
+
 # Non-Nomad Use:
 
 ## Local system installation
@@ -185,8 +226,7 @@ $ deadman-check -h
 
   DESCRIPTION:
 
-    Monitor a Consul key that contains an EPOCH time entry.
-      Send a Slack message if EPOCH age hits given threshold
+    Monitor a Consul key or key-path that contains an EPOCH time entry and frequency. Send Slack message if EPOCH age is greater than given frequency
 
   COMMANDS:
 
@@ -221,12 +261,12 @@ $ deadman-check key_set -h
 
   DESCRIPTION:
 
-
+    key_set will set a consul key that contains the current epoch and time frequency that job should be running at, example key {"epoch":1493010437,"frequency":"300"}
 
   EXAMPLES:
 
     # Update a Consul key deadman/myservice, with current EPOCH time
-    deadman-check key_set --host 127.0.0.1 --port 8500 --key deadman/myservice
+    deadman-check key_set --host 127.0.0.1 --port 8500 --key deadman/myservice --frequency 300
 
   OPTIONS:
 
@@ -237,7 +277,10 @@ $ deadman-check key_set -h
         port Consul is listening on
 
     --key KEY
-        Consul key to monitor
+        Consul key to report EPOCH time and frequency for service
+
+    --frequency FREQUENCY
+        Frequency at which this key should be updated in seconds
 ```
 
 ### Usage for switch_monitor command
@@ -255,18 +298,15 @@ $ deadman-check switch_monitor -h
 
   DESCRIPTION:
 
-
+    switch_monitor will monitor either a given key which contains a services last epoch checkin and frequency, or a series of services that set keys under a given key-path in Consul
 
   EXAMPLES:
 
-    # Target a Consul key deadman/myservice, and this key has an EPOCH
-     value to check looking to alert on 500 second or greater freshness
-    deadman-check switch_monitor \
-      --host 127.0.0.1 \
-      --port 8500 \
-      --key deadman/myservice \
-      --freshness 500 \
-      --alert-to slackroom
+    # Target a Consul key deadman/myservice, and this key has an EPOCH value to check looking to alert
+    deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --alert-to monitoroom
+
+    # Target a Consul key path deadman/, which contains 2 or more service keys to monitor, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path deadman/
+    deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to monitoroom
 
   OPTIONS:
 
@@ -276,15 +316,14 @@ $ deadman-check switch_monitor -h
     --port PORT
         port Consul is listening on
 
-    --key KEY
-        Consul key to monitor
+    --key-path KEYPATH
+        Consul key path to monitor, performs a recursive key lookup at given path.
 
-    --freshness SECONDS
-        The value in seconds to alert on when the recorded
-            EPOCH value exceeds current EPOCH
+    --key KEY
+        Consul key to monitor, provide this or --key-path if you have multiple keys in a given path.
 
     --alert-to SLACKROOM
-        Slackroom to alert to, don't include the # tag in name
+        Slackroom to send alert, don't include the # tag in name
 
     --daemon
         Run as a daemon, otherwise will run check just once

data/bin/deadman-check

@@ -7,26 +7,37 @@ require 'daemons'
 
 program :name, 'deadman-check'
 program :version, DeadmanCheck::VERSION
-program :description, %q{Monitor a Consul key that contains an EPOCH time entry. Send email if EPOCH age hits given threshold}
+program :description, %q{Monitor a Consul key or key-path that contains an EPOCH time entry and frequency. Send Slack message if EPOCH age is greater than given frequency}
 
 command :switch_monitor do |c|
   c.syntax = 'deadman-check switch_monitor [options]'
   c.summary = 'Target a Consul key to monitor'
-  c.description = ''
-  c.example %q{Target a Consul key deadman/myservice, and this key has an EPOCH value to check looking to alert on 500 second or greater freshness},
-    %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --freshness 500 --alert-to ops@mycomany.tld --alert-from ops-no-reply-email@mycomany.tld}
+  c.description = 'switch_monitor will monitor either a given key which contains a services last epoch checkin and frequency, or a series of services that set keys under a given key-path in Consul'
+  c.example %q{Target a Consul key deadman/myservice, and this key has an EPOCH value to check looking to alert},
+    %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --alert-to monitoroom}
+  c.example %q{Target a Consul key path deadman/, which contains 2 or more service keys to monitor, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path deadman/},
+    %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to monitoroom}
   c.option '--host HOST', String, 'IP address or hostname of Consul system'
   c.option '--port PORT', String, 'port Consul is listening on'
-  c.option '--key KEY', String, 'Consul key to monitor'
-  c.option '--freshness SECONDS', String, %q{The value in seconds to alert on when the recorded
-    EPOCH value exceeds current EPOCH}
-  c.option '--alert-to SLACKROOM', String, 'Slackroom to alert to'
+  c.option '--key-path KEYPATH', String, 'Consul key path to monitor, performs a recursive key lookup at given path.'
+  c.option '--key KEY', String, 'Consul key to monitor, provide this or --key-path if you have multiple keys in a given path.'
+  c.option '--alert-to SLACKROOM', String, 'Slackroom to send alert, don\'t include the # tag in name'
   c.option '--daemon', 'Run as a daemon, otherwise will run check just once'
   c.option '--daemon-sleep SECONDS', String, 'Set the number of seconds to sleep in between switch checks, default 300'
   c.action do |args, options|
     options.default :daemon_sleep => 300
+    if options.key_path && options.key
+      abort("Specify --key-path or --key, don't specify both")
+    end
+    if options.key
+      target = options.key
+      recurse = false
+    else
+      target = options.key_path
+      recurse = true
+    end
     switch_monitor = DeadmanCheck::SwitchMonitor.new(options.host, options.port,
-      options.key, options.freshness, options.alert_to, options.daemon_sleep)
+      target, options.alert_to, recurse, options.daemon_sleep)
     if options.daemon
       Daemons.run(switch_monitor.run_check_daemon)
     else
@@ -38,15 +49,16 @@ end
 command :key_set do |c|
   c.syntax = 'deadman-check key_set [options]'
   c.summary = 'Update a given Consul key with current EPOCH'
-  c.description = ''
+  c.description = 'key_set will set a consul key that contains the current epoch and time frequency for which the job being monitored runs at, example key {"epoch":1493010437,"frequency":"300"}'
   c.example %q{Update a Consul key deadman/myservice, with current EPOCH time},
-    %q{deadman-check key_set --host 127.0.0.1 --port 8500 --key deadman/myservice}
+    %q{deadman-check key_set --host 127.0.0.1 --port 8500 --key deadman/myservice --frequency 300}
   c.option '--host HOST', String, 'IP address or hostname of Consul system'
   c.option '--port PORT', String, 'port Consul is listening on'
-  c.option '--key KEY', String, 'Consul key to report EPOCH time to'
+  c.option '--key KEY', String, 'Consul key to report EPOCH time and frequency for service'
+  c.option '--frequency FREQUENCY', String, 'Frequency at which this key should be updated in seconds'
   c.action do |args, options|
-    key_set = DeadmanCheck::KeySet.new(options.host, options.port,
-      options.key)
+    key_set = DeadmanCheck::KeySet.new(options.host, options.port, options.key,
+      options.frequency)
     key_set.run_consul_key_update
   end
 end

data/deadman_check.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "webmock", "~> 3.0"
 
   spec.add_dependency 'commander', '~> 4.4', '>= 4.4.3'
   spec.add_dependency 'diplomat', '~> 1.2', '>= 1.2.0'

data/lib/deadman_check/version.rb

@@ -1,3 +1,3 @@
 module DeadmanCheck
-  VERSION = "0.1.11"
+  VERSION = "0.2.0"
 end

data/lib/deadman_check_keyset.rb

@@ -1,33 +1,35 @@
 require 'deadman_check/version'
 require 'deadman_check_global'
 require 'diplomat'
+require 'json'
 
 module DeadmanCheck
   # KeySet Class
   class KeySet
-    attr_accessor :host, :port, :key
+    attr_accessor :host, :port, :key, :frequency
 
-    def initialize(host, port, key)
+    def initialize(host, port, key, frequency)
       @host = host
       @port = port
       @key  = key
+      @frequency = frequency
     end
 
-    def _configure_diplomat(host, port)
-      Diplomat.configure do |config|
-        config.url = "http://#{host}:#{port}"
-      end
+    def run_consul_key_update
+      update_consul_key(@host, @port, @key, @frequency)
     end
 
-    def _update_consul_key(host, port, key)
-      DeadmanCheck::DeadmanCheckGlobal.new.configure_diplomat(host, port)
-      epoch_time_now = DeadmanCheck::DeadmanCheckGlobal.new.get_epoch_time
-      Diplomat::Kv.put(key, "#{epoch_time_now}")
-      puts "Consul key #{key} updated EPOCH to #{epoch_time_now}"
-    end
+    private
+      def generate_json(epoch, frequency)
+        consul_key = { :epoch => epoch, :frequency => frequency }
+        consul_key.to_json
+      end
 
-    def run_consul_key_update
-      _update_consul_key(@host, @port, @key)
-    end
+      def update_consul_key(host, port, key, frequency)
+        DeadmanCheck::DeadmanCheckGlobal.new.configure_diplomat(host, port)
+        epoch_time_now = DeadmanCheck::DeadmanCheckGlobal.new.get_epoch_time
+        Diplomat::Kv.put(key, "#{generate_json(epoch_time_now, frequency)}")
+        puts "Consul key #{key} updated EPOCH to #{epoch_time_now}"
+      end
   end
 end

data/lib/deadman_check_switch.rb

@@ -2,18 +2,19 @@ require 'deadman_check/version'
 require 'deadman_check_global'
 require 'diplomat'
 require 'slack-ruby-client'
+require 'json'
 
 module DeadmanCheck
   # Switch class
   class SwitchMonitor
-    attr_accessor :host, :port, :key, :freshness, :alert_to, :daemon_sleep
+    attr_accessor :host, :port, :target, :alert_to, :recurse, :daemon_sleep
 
-    def initialize(host, port, key, freshness, alert_to, daemon_sleep)
+    def initialize(host, port, target, alert_to, recurse, daemon_sleep)
       @host = host
       @port = port
-      @key  = key
-      @freshness = freshness.to_i
+      @target = target
       @alert_to = alert_to
+      @recurse = recurse
       @daemon_sleep = daemon_sleep.to_i
     end
 
@@ -21,39 +22,74 @@ module DeadmanCheck
       config.token = ENV['SLACK_API_TOKEN']
     end
 
-    def _diff_epoc(current_epoch, recorded_epoch)
-      epoch_difference = current_epoch - recorded_epoch
-      return epoch_difference
-    end
-
-    def _get_recorded_epoch(host, port, key)
-      DeadmanCheck::DeadmanCheckGlobal.new.configure_diplomat(host, port)
-      recorded_epoch = Diplomat::Kv.get(key)
-      return recorded_epoch
-    end
-
-    def slack_alert(alert_to, key, epoch_diff)
-      client = Slack::Web::Client.new
-      client.chat_postMessage(channel: "\##{alert_to}", text: "Alert: Deadman Switch
-        Triggered for #{key}, with #{epoch_diff} seconds since last run",
-        username: 'deadman')
-    end
-
     def run_check_once
-      recorded_epoch = _get_recorded_epoch(@host, @port, @key).to_i
+      recorded_epochs = get_recorded_epochs(@host, @port, @target, @recurse)
       current_epoch = DeadmanCheck::DeadmanCheckGlobal.new.get_epoch_time.to_i
-      epoch_diff = _diff_epoc(current_epoch, recorded_epoch)
-      if epoch_diff > @freshness
-        slack_alert(@alert_to, @key, epoch_diff)
+      if @recurse
+        check_recursive_recorded_epochs(recorded_epochs, current_epoch)
+      else
+        record = parse_recorded_epoch(recorded_epochs)
+        check_recorded_epoch(record, current_epoch)
       end
     end
 
     def run_check_daemon
       loop do
-        run_check_once
+        run_check_once(target, recurse)
         sleep(@daemon_sleep)
       end
     end
 
+    private
+      def diff_epoch(current_epoch, recorded_epoch)
+        epoch_difference = current_epoch - recorded_epoch
+        return epoch_difference
+      end
+
+      def get_recorded_epochs(host, port, target, recurse)
+        DeadmanCheck::DeadmanCheckGlobal.new.configure_diplomat(host, port)
+        recorded_epochs = Diplomat::Kv.get(target, recurse: recurse)
+        return recorded_epochs
+      end
+
+      def parse_recorded_epoch(recorded_epochs)
+        # {"epoch":1493000501,"frequency":"300"}
+        value_json = JSON.parse(recorded_epochs)
+        frequency = value_json["frequency"].to_i
+        epoch = value_json["epoch"].to_i
+        return epoch, frequency
+      end
+
+      def alert_if_epoch_greater_than_frequency(epoch_diff, target, frequency)
+        if epoch_diff > frequency
+          slack_alert(@alert_to, target, epoch_diff)
+        end
+      end
+
+      def check_recorded_epoch(parse_recorded_epoch, current_epoch)
+        recorded_epoch = parse_recorded_epoch[0].to_i
+        frequency = parse_recorded_epoch[1].to_i
+        epoch_diff = diff_epoch(current_epoch, recorded_epoch)
+        alert_if_epoch_greater_than_frequency(epoch_diff, @target, frequency)
+      end
+
+      def check_recursive_recorded_epochs(recorded_epochs, current_epoch)
+        recorded_epochs.each do |recorded_service|
+          value_json = JSON.parse(recorded_service[:value])
+          frequency = value_json["frequency"].to_i
+          epoch = value_json["epoch"].to_i
+          epoch_diff = diff_epoch(current_epoch, epoch)
+          alert_if_epoch_greater_than_frequency(epoch_diff,
+                                                recorded_service[:key],
+                                                frequency)
+        end
+      end
+
+      def slack_alert(alert_to, target, epoch_diff)
+        client = Slack::Web::Client.new
+        client.chat_postMessage(channel: "\##{alert_to}", text: "Alert: Deadman Switch
+          Triggered for #{target}, with #{epoch_diff} seconds since last run",
+          username: 'deadman')
+      end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deadman_check
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.2.0
 platform: ruby
 authors:
 - zane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-13 00:00:00.000000000 Z
+date: 2017-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: commander
   requirement: !ruby/object:Gem::Requirement