deadman_check 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d5b9a2213bd31ca4cbc8c63386e77690b6719050
-  data.tar.gz: e28c30d37596322c9407be8db5b115cecae6b8c5
+  metadata.gz: 59091e4397024d50031cf4368adfea22a4057207
+  data.tar.gz: dcc1a25d10b623e2b4e13c928ba19cbef1ccc1e9
 SHA512:
-  metadata.gz: 6901e0ed28c8ca36dcbdb6e1dd391713fe2cf5f1477379c4562e0bebdf3f0bea1f3d201e93e640f4d1ea9aaa02ea7570c3d5c41297da6a68638335bbda68af72
-  data.tar.gz: c0eb413dd8657997489f0730515243fc30981a89359019999df8a738fa06112d976cfd2233447173305b5f8493199eebb92e47ff61e162f2d2d9cd35e066786d
+  metadata.gz: 394b714ccbb820e3fc3e526baadc3bb75ef51309ba3a08cd058548cbe8c6b4fe5da7219182023a523d9db19438319a42f30ff7e39d92098bf93bbe8f1c9f1ae0
+  data.tar.gz: bfc91fcadb9bb51475fd7382516dc687adce7d3c5917e5fa8b6bb26989432301e17ae9d06ee7fed3e74332a6eaa0ca1a6d6b984e6371a2439dd480c861f66e2b

data/README.md

@@ -18,8 +18,11 @@ is expected for that job.
 
 
 ### Requirements
-* [Consul](https://www.consul.io/) instance
-* Alerting requires a SLACK_API_TOKEN environment variable to be set (use [Slack Bot integration](https://my.slack.com/services/new/bot))
+* [Consul](https://www.consul.io/) instance or cluster to report to
+
+### Alerting Options
+* [Slack](https://slack.com/) 
+* [AWS SNS](https://aws.amazon.com/documentation/sns/)
 
 ## Example Usage
 
@@ -127,7 +130,7 @@ job "DeadmanMonitoring" {
           "8500",
           "--key",
           "deadman/SilverBulletPeriodicProcess",
-          "--alert-to",
+          "--alert-to-slack",
           "slackroom",
           "--daemon",
           "--daemon-sleep",
@@ -153,7 +156,7 @@ If you have multiple periodic jobs that need to be monitored then use the ```--k
 
 <img width="658" alt="screen shot 2017-04-23 at 11 17 29 pm" src="https://cloud.githubusercontent.com/assets/538171/25324510/14d6e7f0-287b-11e7-9c0d-733d69e1cc94.png">
 
-To monitor the above you would just use the ```--key-path``` argument instead of ```--key```
+To monitor the above you would just use the ```--key-path``` argument instead of ```--key``` and AWS SNS for alerting endpoint
 
 ```hcl
 job "DeadmanMonitoring" {
@@ -173,8 +176,10 @@ job "DeadmanMonitoring" {
           "8500",
           "--key-path",
           "deadman/",
-          "--alert-to",
-          "slackroom",
+          "--alert-to-sns",
+          "arn:aws:sns:us-east-1:123412345678:deadman-check",
+          "--alert-to-sns-region",
+          "us-east-1",
           "--daemon",
           "--daemon-sleep",
           "900"]
@@ -184,13 +189,16 @@ job "DeadmanMonitoring" {
         memory = 256
       }
       env {
-        SLACK_API_TOKEN = "YourSlackApiToken"
+        AWS_ACCESS_KEY_ID = "YourAWSKEY"
+        AWS_SECRET_ACCESS_KEY = "YourAWSSecret"
       }
     }
   }
 }
 ```
 
+<img width="903" alt="screen shot 2017-08-04 at 11 39 12 am" src="https://user-images.githubusercontent.com/538171/28982223-e576743c-7909-11e7-8e65-ebb0b4a76762.png">
+
 # Non-Nomad Use:
 
 ## Local system installation
@@ -216,6 +224,13 @@ $ alias deadman-check='\
 
 If you don't do the docker pull, the first time you run deadman-check, the docker run command will automatically pull the sepulworld/deadman-check image on the Docker Hub. Subsequent runs will use a locally cached copy of the image and will not have to download anything.
 
+### Alerting Setup
+* Slack alerting requires a SLACK_API_TOKEN environment variable to be set (use [Slack Bot integration](https://my.slack.com/services/new/bot)) (optional)
+* [AWS SNS](https://aws.amazon.com/documentation/sns/) alerting requires appropreiate AWS IAM access to target SNS topic. One of the following can be used for authentication. IAM policy access to publish to the topic will be required
+  - ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY']
+  - The shared credentials ini file at ~/.aws/credentials (more information)
+  - From an [instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) when running on EC2
+
 ## Usage via Local System Install
 
 ```bash
@@ -298,15 +313,21 @@ $ deadman-check switch_monitor -h
 
   DESCRIPTION:
 
-    switch_monitor will monitor either a given key which contains a services last epoch checkin and frequency, or a series of services that set keys under a given key-path in Consul
+    switch_monitor will monitor either a given key which contains a services last epoch checkin and frequency, or a series of services that set keys
+under a given key-path in Consul
 
   EXAMPLES:
 
     # Target a Consul key deadman/myservice, and this key has an EPOCH value to check looking to alert
-    deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --alert-to monitoroom
+    deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --alert-to-slack my-slack-monitor-channel
+
+    # Target a Consul key path deadman/, which contains 2 or more service keys to monitor, i.e. deadman/myservice1, deadman/myservice2,
+deadmman/myservice3 all fall under the path deadman/
+    deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to-slack my-slack-monitor-channel
 
-    # Target a Consul key path deadman/, which contains 2 or more service keys to monitor, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path deadman/
-    deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to monitoroom
+    # Target a Consul key path deadman/, alert to Amazon SNS, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path
+deadman/
+    deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to-sns arn:aws:sns:*:123456789012:my_corporate_topic
 
   OPTIONS:
 
@@ -322,8 +343,14 @@ $ deadman-check switch_monitor -h
     --key KEY
         Consul key to monitor, provide this or --key-path if you have multiple keys in a given path.
 
-    --alert-to SLACKROOM
-        Slackroom to send alert, don't include the # tag in name
+    --alert-to-slack SLACKCHANNEL
+        Slack channel to send alert, don't include the # tag in name
+
+    --alert-to-sns SNSARN
+        Amazon Web Services SNS arn to send alert, example arn arn:aws:sns:*:123456789012:my_corporate_topic
+
+    --alert-to-sns-region AWSREGION
+        Amazon Web Services region the SNS topic is in, defaults to us-west-2
 
     --daemon
         Run as a daemon, otherwise will run check just once

data/Rakefile

@@ -5,6 +5,7 @@ Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "lib"
   t.test_files = FileList['test/**/*_test.rb']
+  t.warning = false
 end
 
 task :default => :test

data/bin/deadman-check

@@ -14,18 +14,26 @@ command :switch_monitor do |c|
   c.summary = 'Target a Consul key to monitor'
   c.description = 'switch_monitor will monitor either a given key which contains a services last epoch checkin and frequency, or a series of services that set keys under a given key-path in Consul'
   c.example %q{Target a Consul key deadman/myservice, and this key has an EPOCH value to check looking to alert},
-    %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --alert-to monitoroom}
+    %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key deadman/myservice --alert-to-slack my-slack-monitor-channel}
   c.example %q{Target a Consul key path deadman/, which contains 2 or more service keys to monitor, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path deadman/},
-    %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to monitoroom}
+    %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to-slack my-slack-monitor-channel}
+  c.example %q{Target a Consul key path deadman/, alert to Amazon SNS, i.e. deadman/myservice1, deadman/myservice2, deadmman/myservice3 all fall under the path deadman/},
+    %q{deadman-check switch_monitor --host 127.0.0.1 --port 8500 --key-path deadman/ --alert-to-sns arn:aws:sns:*:123456789012:my_corporate_topic}
   c.option '--host HOST', String, 'IP address or hostname of Consul system'
   c.option '--port PORT', String, 'port Consul is listening on'
   c.option '--key-path KEYPATH', String, 'Consul key path to monitor, performs a recursive key lookup at given path.'
   c.option '--key KEY', String, 'Consul key to monitor, provide this or --key-path if you have multiple keys in a given path.'
-  c.option '--alert-to SLACKROOM', String, 'Slackroom to send alert, don\'t include the # tag in name'
+  c.option '--alert-to-slack SLACKCHANNEL', String, 'Slack channel to send alert, don\'t include the # tag in name'
+  c.option '--alert-to-sns SNSARN', String, 'Amazon Web Services SNS arn to send alert, example arn arn:aws:sns:*:123456789012:my_corporate_topic'
+  c.option '--alert-to-sns-region AWSREGION', String, 'Amazon Web Services region the SNS topic is in, defaults to us-west-2'
   c.option '--daemon', 'Run as a daemon, otherwise will run check just once'
   c.option '--daemon-sleep SECONDS', String, 'Set the number of seconds to sleep in between switch checks, default 300'
   c.action do |args, options|
-    options.default :daemon_sleep => 300
+    options.default :daemon_sleep => 300,
+                    :alert_to_sns_region => 'us-west-2',
+                    :alert_to_sns => nil,
+                    :alert_to_slack => nil
+
     if options.key_path && options.key
       abort("Specify --key-path or --key, don't specify both")
     end
@@ -36,8 +44,10 @@ command :switch_monitor do |c|
       target = options.key_path
       recurse = true
     end
-    switch_monitor = DeadmanCheck::SwitchMonitor.new(options.host, options.port,
-      target, options.alert_to, recurse, options.daemon_sleep)
+    switch_monitor = DeadmanCheck::SwitchMonitor.new(
+      options.host, options.port,
+      target, options.alert_to_slack, options.alert_to_sns,
+      options.alert_to_sns_region, recurse, options.daemon_sleep)
     if options.daemon
       Daemons.run(switch_monitor.run_check_daemon)
     else
@@ -57,6 +67,12 @@ command :key_set do |c|
   c.option '--key KEY', String, 'Consul key to report EPOCH time and frequency for service'
   c.option '--frequency FREQUENCY', String, 'Frequency at which this key should be updated in seconds'
   c.action do |args, options|
+    if options.frequency.nil?
+      abort("Specify --frequency at which this key should be updated by the service")
+    end
+    if options.key.nil?
+      abort("Must specify a --key")
+    end
     key_set = DeadmanCheck::KeySet.new(options.host, options.port, options.key,
       options.frequency)
     key_set.run_consul_key_update

data/deadman_check.gemspec

@@ -36,7 +36,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "webmock", "~> 3.0"
 
   spec.add_dependency 'commander', '~> 4.4', '>= 4.4.3'
-  spec.add_dependency 'diplomat', '~> 1.2', '>= 1.2.0'
+  spec.add_dependency 'diplomat', '~> 2.0.0', '>= 2.0.0'
   spec.add_dependency 'slack-ruby-client', '~> 0.8.0'
   spec.add_dependency 'daemons', '~> 1.2.4', '>=1.2.4'
+  spec.add_dependency 'aws-sdk', '~> 2.10.21', '>=2.10.21'
 end

data/lib/deadman_check/version.rb

@@ -1,3 +1,3 @@
 module DeadmanCheck
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/deadman_check_switch.rb

@@ -3,23 +3,36 @@ require 'deadman_check_global'
 require 'diplomat'
 require 'slack-ruby-client'
 require 'json'
+require 'aws-sdk'
 
 module DeadmanCheck
   # Switch class
   class SwitchMonitor
-    attr_accessor :host, :port, :target, :alert_to, :recurse, :daemon_sleep
+    attr_accessor :host, :port, :target, :alert_to_slack,
+    :alert_to_sns, :alert_to_sns_region, :recurse, :daemon_sleep
 
-    def initialize(host, port, target, alert_to, recurse, daemon_sleep)
+    def initialize(host, port, target, alert_to_slack, alert_to_sns,
+                  alert_to_sns_region, recurse, daemon_sleep)
       @host = host
       @port = port
       @target = target
-      @alert_to = alert_to
+      @alert_to_slack = alert_to_slack
+      @alert_to_sns = alert_to_sns
+      @alert_to_sns_region = alert_to_sns_region
       @recurse = recurse
       @daemon_sleep = daemon_sleep.to_i
-    end
 
-    Slack.configure do |config|
-      config.token = ENV['SLACK_API_TOKEN']
+      unless @alert_to_slack.nil?
+        Slack.configure do |config|
+          config.token = ENV['SLACK_API_TOKEN']
+        end
+      end
+
+      unless @alert_to_sns.nil?
+        @sns = Aws::SNS::Client.new(
+          region: @alert_to_sns_region
+          )
+      end
     end
 
     def run_check_once
@@ -52,20 +65,26 @@ module DeadmanCheck
         return recorded_epochs
       end
 
+      def check_recursive_recorded_epochs(recorded_epochs, current_epoch)
+        recorded_epochs.each do |recorded_service|
+          value_json = JSON.parse(recorded_service[:value])
+          frequency = value_json["frequency"].to_i
+          epoch = value_json["epoch"].to_i
+          epoch_diff = diff_epoch(current_epoch, epoch)
+          alert_if_epoch_greater_than_frequency(epoch_diff,
+                                                recorded_service[:key],
+                                                frequency)
+        end
+      end
+
       def parse_recorded_epoch(recorded_epochs)
         # {"epoch":1493000501,"frequency":"300"}
-        value_json = JSON.parse(recorded_epochs)
-        frequency = value_json["frequency"].to_i
-        epoch = value_json["epoch"].to_i
+        value_json = JSON.parse(recorded_epochs[0][:value])
+        frequency = value_json["frequency"]
+        epoch = value_json["epoch"]
         return epoch, frequency
       end
 
-      def alert_if_epoch_greater_than_frequency(epoch_diff, target, frequency)
-        if epoch_diff > frequency
-          slack_alert(@alert_to, target, epoch_diff)
-        end
-      end
-
       def check_recorded_epoch(parse_recorded_epoch, current_epoch)
         recorded_epoch = parse_recorded_epoch[0].to_i
         frequency = parse_recorded_epoch[1].to_i
@@ -73,23 +92,34 @@ module DeadmanCheck
         alert_if_epoch_greater_than_frequency(epoch_diff, @target, frequency)
       end
 
-      def check_recursive_recorded_epochs(recorded_epochs, current_epoch)
-        recorded_epochs.each do |recorded_service|
-          value_json = JSON.parse(recorded_service[:value])
-          frequency = value_json["frequency"].to_i
-          epoch = value_json["epoch"].to_i
-          epoch_diff = diff_epoch(current_epoch, epoch)
-          alert_if_epoch_greater_than_frequency(epoch_diff,
-                                                recorded_service[:key],
-                                                frequency)
+      def alert_if_epoch_greater_than_frequency(epoch_diff, target, frequency)
+        if epoch_diff > frequency
+          slack_alert(
+            @alert_to_slack, target, epoch_diff) unless @alert_to_slack.nil?
+          sns_alert(
+            @alert_to_sns, target, epoch_diff) unless @alert_to_sns.nil?
         end
       end
 
-      def slack_alert(alert_to, target, epoch_diff)
+      def slack_alert(alert_to_slack, target, epoch_diff)
         client = Slack::Web::Client.new
-        client.chat_postMessage(channel: "\##{alert_to}", text: "Alert: Deadman Switch
+        client.chat_postMessage(channel: "\##{alert_to_slack}",
+          text: "Alert: Deadman Switch
           Triggered for #{target}, with #{epoch_diff} seconds since last run",
           username: 'deadman')
       end
+
+      def sns_alert(alert_to_sns, target, epoch_diff)
+        @sns.publish(
+          target_arn: @alert_to_sns,
+          message_structure: 'json',
+          message: {
+            :default => "Alert: Deadman Switch triggered for #{target}",
+            :email => "Alert: Deadman Switch triggered for #{target}, with
+            #{epoch_diff} seconds since last run",
+            :sms => "Alert: Deadman Switch for #{target}"
+          }.to_json
+        )
+      end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deadman_check
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - zane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-25 00:00:00.000000000 Z
+date: 2017-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -92,20 +92,20 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: 2.0.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: 2.0.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: slack-ruby-client
   requirement: !ruby/object:Gem::Requirement
@@ -140,6 +140,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.4
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.21
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.10.21
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.10.21
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.10.21
 description: |-
   A script to check a given Consul key EPOCH for
       freshness. Good for monitoring cron jobs or batch jobs. Have the last step