bosh-monitor 1.2980.0 → 1.2981.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5b153d8361a68b7be9246dca36ea5c91e791ba2
-  data.tar.gz: 55ed9a88a052175affce4963dc78f49fb13fce07
+  metadata.gz: de5c0802c02e2abd67ade2f77f300ef6e8438e9a
+  data.tar.gz: 4f2e36dc946ccab2afa26fb66f517fedf03bd97b
 SHA512:
-  metadata.gz: 3f9e46d1751ae95e516005560eb809282e6e683e1ada1bc0ec9a4974cbad63753fbf7d139a5f88481a1cd3972975e833ffa704a35a707522b44976a5d4f32f16
-  data.tar.gz: b6178d2945aca5d646ca4505288c2f15b06b269053c3c4b69d973c5c4acd50f82193ea94cfdefe6e1a20cf9f7152cee36d7b9297d10caf25240edae0f56fe7d2
+  metadata.gz: 1e883ab6db000252c6b5735f32724e1076a127ca847986cbf55d8035a3f325ee4155b43aaa8c2fb6c64f48c80a76cb0f3e38a64833a9a911c58ba730e78396e7
+  data.tar.gz: 2c1160cbf398c4b57e8b338b7d5547f1617f298aa7f5412156f3c45f6835ea2b01d036c33579252f32564eb4b6c63dfd9474dcb3ec70e6032d9d3df13b145de3

data/README.md

@@ -45,7 +45,8 @@ Among the included plugins are:
 - DataDog - Sends various events to DataDog.com using their API
 - AWS CloudWatch - Sends various events to Amazon's CloudWatch using their API
 - Emailer - Sends configurable Emails on events reciept
-
+  Consul Event Forwarder - Sends heartbeats as events and TTL checks to a consul cluster
+  
 Plugins should conform to the following interface:
 
 | *Method* | *Arguments* | *Description* |

data/lib/bosh/monitor.rb

@@ -74,3 +74,4 @@ require 'bosh/monitor/plugins/pagerduty'
 require 'bosh/monitor/plugins/resurrector'
 require 'bosh/monitor/plugins/tsdb'
 require 'bosh/monitor/plugins/varz'
+require 'bosh/monitor/plugins/consul_event_forwarder'

data/lib/bosh/monitor/plugins/README.md

@@ -0,0 +1,57 @@
+#Using Bosh Monitor Plugins
+
+##AWS CloudWatch
+Sends various events to Amazon's CloudWatch using their
+
+##DataDog
+Sends various events to DataDog.com using their API
+
+| option           | description            |
+|------------------|------------------------|
+| api_key          | Your api Key           |
+| application_key  | Your Application Key   |
+
+##Consul Event Forwarder Plugin
+The Consul plugin works by forwarding nats heartbeat events and alerts to a consul server or agent. The nats messages can be forwarded as ttl checks and events. Heartbeat messages will be forwarded as TTL checks, each time a heartbeat occurs it will update the ttl check with it's status. When an alert occurs it will be forwareded to Consul as an Event. The current best use case seems to be to forward to a consul agent (possibly on your inception server)
+
+| option                | description                         |
+|-----------------------|-------------------------------------|
+|  host                 | The address of the cluster or agent |
+|  namespace            | A namespace to separate multiple instances of the same release
+|  events_api           | The events api endpoint defaults to /v1/event/fire/
+|  ttl_api              | The Check update and registration endpoint defaults to /v1/agent/check/
+|  port                 | Defaults to 8500
+|  protocal             | Defaults to HTTP
+|  params               | Can be used to pass access token "token=MYACCESSTOKEN"
+|  ttl                  | TTL Checks will be used if a ttl period is set here. Example "120s"
+|  events               | If set to true heartbeats will be forwarded as events
+|  ttl_note             | A note that will be passed back to consul with a ttl check
+|  heartbeats_as_alerts | * If set to true all heartbeats will also be forwarded as event, this gives you 'real time' vitals data to correlate with
+
+####* When heartbeats are sent as alerts the format has been made more concise to come in under the event payload bytesize limits that consul enforces
+```ruby
+{
+  :agent  => agent_id,
+  :name   => "job_name / index",
+  :state  => job_state,
+  :data   => {
+      :cpu => [sys, user, wait]
+      :dsk => {
+        :eph => [inode_percent, percent],
+        :sys =>[inode_percent, percent]
+      }
+      :ld  => load,
+      :mem => [kb, percent],
+      :swp => [kb, percent]
+  }
+}
+```
+
+## Event Logger
+ Logs all events
+
+## PagerDuty
+Sends various events to PagerDuty.com using their API
+
+## Resurrector
+Restarts VMs that have stopped heartbeating

data/lib/bosh/monitor/plugins/consul_event_forwarder.rb

@@ -0,0 +1,170 @@
+# Consul Bosh Monitor Plugin
+# Forwards alert and heartbeat messages as events to a consul agent
+module Bosh::Monitor
+  module Plugins
+    class ConsulEventForwarder < Base
+      include Bosh::Monitor::Plugins::HttpRequestHelper
+
+      CONSUL_REQUEST_HEADER  = { 'Content-Type' => 'application/javascript' }
+      TTL_STATUS_MAP         = { 'running' => :pass, 'failing' => :fail, 'unknown' => :fail, 'default' => :warn }
+      REQUIRED_OPTIONS       = ["host", "port", "protocol" ]
+      CONSUL_MAX_EVENT_BYTESIZE = 512
+
+      CONSUL_ENDPOINTS = {
+        event:     "/v1/event/fire/",              #fire an event
+        register:   "/v1/agent/check/register",    #register a check
+        deregister: "/v1/agent/check/deregister/", #deregister a check
+        pass:       "/v1/agent/check/pass/",       #mark a check as passing
+        warn:       "/v1/agent/check/warn/",       #mark a check as warning
+        fail:       "/v1/agent/check/fail/"        #mark a check as failing
+      }
+
+      def run
+        @checklist       = []
+        @host            = options['host']
+        @namespace       = options['namespace']
+        @port            = options['port']
+        @protocol        = options['protocol']
+        @params          = options['params']
+        @ttl             = options['ttl']
+        @use_events      = options['events']
+        @ttl_note        = options['ttl_note']
+
+        @heartbeats_as_alerts = options['heartbeats_as_alerts']
+        @use_ttl              = !@ttl.nil?
+
+        @status_map = Hash.new(:warn)
+        @status_map.merge!(TTL_STATUS_MAP)
+
+        logger.info("Consul Event Forwarder plugin is running...")
+      end
+
+      def validate_options
+        valid_array = REQUIRED_OPTIONS.map{ |o| options[o].to_s.empty? }
+        !valid_array.include?(true)
+      end
+
+      def process(event)
+        validate_options && forward_event(event)
+      end
+
+      private
+
+      def consul_uri(event, note_type)
+        path = get_path_for_note_type(event, note_type)
+        URI.parse("#{@protocol}://#{@host}:#{@port}#{path}?#{@params}")
+      end
+
+      #heartbeats get forwarded as ttl checks and alerts get forwarded as events
+      #if heartbeat_as_alert is true than a heartbeat gets forwarded as events as well
+      def forward_event(event)
+
+        if forward_this_event?(event)
+          notify_consul(event, :event)
+        end
+
+        if forward_this_ttl?(event)
+          event_unregistered?(event) ? notify_consul(event, :register, registration_payload(event)) : notify_consul(event, :ttl)
+        end
+
+      end
+
+      #should an individual alert or heartbeat be forwarded as a consul event
+      def forward_this_event?(event)
+        @use_events && ( event.is_a?(Bosh::Monitor::Events::Alert) || ( event.is_a?(Bosh::Monitor::Events::Heartbeat) && @heartbeats_as_alerts) )
+      end
+
+      def forward_this_ttl?(event)
+         @use_ttl && event.is_a?(Bosh::Monitor::Events::Heartbeat)
+      end
+
+      def get_path_for_note_type(event, note_type)
+        case note_type
+        when :event
+          CONSUL_ENDPOINTS[:event] + label_for_event(event)
+        when :ttl
+          job_state = event.attributes['job_state']
+          status_id = @status_map[job_state]
+          CONSUL_ENDPOINTS[status_id] + label_for_ttl(event)
+        when :register
+          CONSUL_ENDPOINTS[:register]
+        end
+      end
+
+      def label_for_event(event)
+        case event
+          when Bosh::Monitor::Events::Heartbeat
+            "#{@namespace}#{event.job}"
+          when Bosh::Monitor::Events::Alert
+            event_label = event.title.downcase.gsub(" ","_")
+            "#{@namespace}#{event_label}"
+          else
+            #Something we haven't encountered yet
+            "#{@namespace}event"
+        end
+      end
+
+      def label_for_ttl(event)
+        "#{@namespace}#{event.job}"
+      end
+
+      # Notify consul of an event
+      # note_type: the type of notice we are sending (:event, :ttl, :register)
+      # message:   an optional body for the message, event.json is used by default
+      def notify_consul(event, note_type, message=nil)
+        body    = message.nil? ? right_sized_body_for_consul(event).to_json : message.to_json
+        uri     = consul_uri(event, note_type)
+
+        request = { :body => body }
+
+        send_http_put_request(uri , request)
+
+        #if a registration request returns without error we log it
+        #we don't want to send extra registrations
+        @checklist << event.job if note_type == :register
+      rescue => e
+        logger.error("Could not forward event to Consul Cluster @#{@host}: #{e.inspect}")
+      end
+
+      #consul limits event payload to < 512 bytes, unfortunately we have to do some pruning so this limit is not as likely to be reached
+      #this is suboptimal but otherwise the event post will fail, and how do we decide what data isn't important?
+      def right_sized_body_for_consul(event)
+        body = event.to_hash
+        if event.is_a?(Bosh::Monitor::Events::Heartbeat)
+          vitals = body[:vitals]
+          #currently assuming the event hash details are always put together in the same order
+          #this should yield consistent results from the values method
+          {
+            :agent  => body[:agent_id],
+            :name   => "#{body[:job]}/#{body[:index]}",
+            :state  => "#{body[:job_state]}",
+            :data   => {
+                :cpu => vitals['cpu'].values,
+                :dsk => {
+                  :eph => vitals['disk']['ephemeral'].values,
+                  :sys => vitals['disk']['system'].values,
+                },
+                :ld  => vitals['load'],
+                :mem => vitals['mem'].values,
+                :swp => vitals['swap'].values
+            }
+          }
+        else
+          body
+        end
+      end
+
+      #Has this process not encountered a specific ttl check yet?
+      #We keep track so we aren't sending superfluous registrations
+      #Only register ttl for events that have a job assigned
+      def event_unregistered?(event)
+        @use_ttl && event.respond_to?(:job) && !@checklist.include?(event.job)
+      end
+
+      def registration_payload(event)
+        { "name"  => label_for_ttl(event), "notes" => @ttl_note, "ttl" => @ttl }
+      end
+
+    end
+  end
+end

data/lib/bosh/monitor/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Monitor
-    VERSION = '1.2980.0'
+    VERSION = '1.2981.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh-monitor
 version: !ruby/object:Gem::Version
-  version: 1.2980.0
+  version: 1.2981.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-02 00:00:00.000000000 Z
+date: 2015-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
@@ -180,7 +180,7 @@ dependencies:
         version: '0'
 description: |-
   BOSH Health Monitor
-  cb6c5e
+  4246cc
 email: support@cloudfoundry.com
 executables:
 - bosh-monitor-console
@@ -207,8 +207,10 @@ files:
 - lib/bosh/monitor/events/base.rb
 - lib/bosh/monitor/events/heartbeat.rb
 - lib/bosh/monitor/metric.rb
+- lib/bosh/monitor/plugins/README.md
 - lib/bosh/monitor/plugins/base.rb
 - lib/bosh/monitor/plugins/cloud_watch.rb
+- lib/bosh/monitor/plugins/consul_event_forwarder.rb
 - lib/bosh/monitor/plugins/datadog.rb
 - lib/bosh/monitor/plugins/dummy.rb
 - lib/bosh/monitor/plugins/email.rb