upstreamstatus 0.2.6 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd0f5883836782fed4233bd0ace4cd5154c4bc31
-  data.tar.gz: c6711826ac9880fafeec8503830d65a8450b59ab
+  metadata.gz: 551dd8502211ecb555bed1af238fdbb134bb822e
+  data.tar.gz: 6f898adcff62a5ea936973848173d6dd12ebbefc
 SHA512:
-  metadata.gz: 9219c273817d132304dca8cfe47403d2f41d2cb5c55af3df67ed16fb7429b69d14a224ff312b7df56110e0af1de392b4cb0739ef41670f008185dc5d46e34024
-  data.tar.gz: 83c547e71abdc2a041d4cdad1b26c37980b51af03c4148bab8db16766e60282e00910d8b603fdb614cdcb61f6106f15f8f62a62502046d26941801beba953fc8
+  metadata.gz: 68d220a9b70a101755492030c8b001fb1aee2fbb0b5f3b9258b3a08a2484b7f04f1cbf30b84f09783c0c17c685d8d9f9188a72d39de0edf401b2286f467d953e
+  data.tar.gz: f46a8d49bfbc90aa324036894b3cc0a9b75bbb291407da17e6fbb0f10f22e023983e2e11ad685b4e8ea29fb10105200d99a2ac67589cd9efe951a5345531ab06

data/lib/upstreamstatus/version.rb

@@ -1,3 +1,3 @@
 class Upstreamstatus
-  VERSION = "0.2.6"
+  VERSION = "0.3.0"
 end

data/lib/upstreamstatus.rb

@@ -9,6 +9,8 @@ require 'json'
 require 'sentry-raven'
 require 'pagerduty'
 require 'time'
+require 'socket'
+require 'byebug'
 
 class Upstreamstatus
   extend Forwardable
@@ -48,24 +50,26 @@ class Upstreamstatus
     end
   end
 
+  def down_hosts
+    current_status['servers']['server'].select { |s| s['status'] != 'up' }
+  end
+
   def run
-    down_hosts =
-      current_status['servers']['server'].select { |s| s['status'] != 'up' }
+    clear_active_alerts
 
-    if down_hosts.empty?
-      clear_active_alerts!
-      exit 0
-    end
+    exit 0 if down_hosts.empty?
 
     puts "Detected down hosts:\n"
     print_hosts down_hosts
     logger.info "Detected down hosts: #{down_hosts.to_json}"
 
     if opts[:notify]
-      notify(
-        'One or more API upstream hosts listed as down',
-        JSON.pretty_generate(down_hosts)
-      )
+      down_hosts.each do |host|
+        notify(
+          "Upstream host #{host['upstream']} listed as down",
+          host
+        )
+      end
     end
     exit 1
   rescue Interrupt => e
@@ -84,73 +88,71 @@ class Upstreamstatus
   end
 
   def current_status
-    return fake_response if opts[:simulate]
+    @current_status ||= begin
+      return fake_response if opts[:simulate]
 
-    r = Unirest.get status_check_url
+      r = Unirest.get status_check_url
 
-    unless (200..299).include?(r.code)
-      fail "Error code: #{r.code}\n" \
-           "Headers: #{r.headers}" \
-           "Body: #{r.body}"
-    end
+      unless (200..299).include?(r.code)
+        fail "Error code: #{r.code}\n" \
+             "Headers: #{r.headers}" \
+             "Body: #{r.body}"
+      end
 
-    r.body
+      r.body
+    end
   end
 
   private
 
-  def pd_incidents
-    @pd_incidents ||= begin
-      r = Unirest.get(
-        "#{pagerduty_api_url}/incidents",
-        parameters: { service: pagerduty_service_id }
+  def notify(msg, host)
+    if active_alert?(host['upstream'])
+      puts "Already an active alert. Not sending anything (message: #{msg})"
+    else
+      puts "Notifying PagerDuty (message: #{msg})"
+      pagerduty.trigger(
+        msg,
+        incident_key: "upstreamstatus #{Socket.gethostname} #{host['upstream']}",
+        client: Socket.gethostname,
+        details: host
       )
-      fail "Result: #{r.inspect}" unless (200..299).include?(r.code)
-      r.body['incidents']
     end
   end
 
-  def fake_response
-    {
-      'servers' => {
-        'total' => 2,
-        'generation' => 99,
-        'server' => [
-          {
-            'index' => 0,
-            'upstream' => 'testupstream',
-            'name' => '10.0.0.1 =>8080',
-            'status' => 'up',
-            'rise' => 10_459,
-            'fall' => 0,
-            'type' => 'http',
-            'port' => 0
-          },
-          {
-            'index' => 1,
-            'upstream' => 'testupstream',
-            'name' => '10.0.0.2 =>8080',
-            'status' => 'down',
-            'rise' => 10_029,
-            'fall' => 0,
-            'type' => 'http',
-            'port' => 0
-          }
-        ]
+  def active_alert?(host)
+    active_alerts.find do |a|
+      a['incident_key'] == "upstreamstatus #{Socket.gethostname} #{host['upstream']}"
+    end
+  end
+
+  def active_alerts
+    @active_alerts ||= active_alerts_paged
+  end
+
+  def active_alerts_paged(offset = 0)
+    r = Unirest.get(
+      "#{pagerduty_api_url}/incidents",
+      parameters: {
+        service: pagerduty_service_id,
+        status: 'triggered,acknowledged',
+        offset: offset
       }
-    }
+    )
+    fail "Result: #{r.inspect}" unless (200..299).include?(r.code)
+    pointer = r.body['limit'] + offset
+    r.body['incidents'] + (pointer < r.body['total'] ? active_alerts_paged(pointer) : [])
   end
 
-  def clear_active_alerts!
+  def clear_active_alerts
     return unless opts[:notify]
-    pd_incidents.each do |i|
-      resolve_incident i['incident_key'] unless i['status'] == 'resolved'
+    active_alerts.reject { |a| down_hosts_incident_keys.include? a['incident_key'] }.each do |a|
+      puts "Resolving incident: #{a['incident_key']}"
+      pagerduty.get_incident(a['incident_key']).resolve
     end
   end
 
-  def resolve_incident(incident_key)
-    puts "Resolving incident: #{incident_key}"
-    pagerduty.get_incident(incident_key).resolve
+  def down_hosts_incident_keys
+    down_hosts.map { |host| "upstreamstatus #{Socket.gethostname} #{host['upstream']}" }
   end
 
   def logger
@@ -161,39 +163,6 @@ class Upstreamstatus
     @pagerduty ||= Pagerduty.new pagerduty_api_key
   end
 
-  def notify(msg, details)
-    if active_alert?(msg)
-      puts "Already an active alert. Not sending anything (message: #{msg})"
-    else
-      puts "Notifying PagerDuty (message: #{msg})"
-      pd = pagerduty.trigger(
-        msg,
-        client: ENV['hostname'],
-        details: details
-      )
-
-      # You may have noticed that this will overwrite any previously active
-      # alerts, regardless of whether they are the "same" alert. This is a known
-      # limitation.
-
-      File.write(
-        '/var/run/active_upstream_alert',
-        {
-          'Incident Key' => pd.incident_key,
-          'Message' => msg,
-          'Details' => details
-        }.to_json
-      )
-    end
-  end
-
-  def active_alert?(msg)
-    pd_incidents.find do |incident|
-      incident['status'] != 'resolved' &&
-        incident['trigger_summary_data']['description'] == msg
-    end
-  end
-
   def opts
     @opts ||= Trollop.options do
       opt :notify,
@@ -223,4 +192,45 @@ class Upstreamstatus
   def defaults
     { 'status_check_url' => 'http://localhost:8069/status?format=json' }
   end
+
+  def fake_response
+    {
+      'servers' => {
+        'total' => 2,
+        'generation' => 99,
+        'server' => [
+          {
+            'index' => 0,
+            'upstream' => 'testupstream0',
+            'name' => '10.0.0.1 =>8080',
+            'status' => 'up',
+            'rise' => 10_459,
+            'fall' => 0,
+            'type' => 'http',
+            'port' => 0
+          },
+          {
+            'index' => 1,
+            'upstream' => 'testupstream1',
+            'name' => '10.0.0.2 =>8080',
+            'status' => 'down',
+            'rise' => 10_029,
+            'fall' => 0,
+            'type' => 'http',
+            'port' => 0
+          },
+          {
+            'index' => 2,
+            'upstream' => 'testupstream2',
+            'name' => '10.0.0.2 =>8080',
+            'status' => 'down',
+            'rise' => 10_029,
+            'fall' => 0,
+            'type' => 'http',
+            'port' => 0
+          }
+        ]
+      }
+    }
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upstreamstatus
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors:
 - Eric Herot
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-12 00:00:00.000000000 Z
+date: 2016-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unirest
@@ -179,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Parse the output of the Nginx Upstream Check plugin