bosh-monitor 1.5.0.pre.1113

data/lib/bosh/monitor/events/heartbeat.rb

@@ -0,0 +1,139 @@
+module Bosh::Monitor
+  module Events
+    class Heartbeat < Base
+
+      CORE_JOBS = Set.new(%w(cloud_controller dea health_manager nats router routerv2 stager uaa vcap_redis))
+
+      SERVICE_JOBS_PREFIXES = %w(mysql mongodb redis rabbit postgresql vblob).join("|")
+      SERVICE_JOBS_GATEWAY_REGEX = /(#{SERVICE_JOBS_PREFIXES})_gateway$/i
+      SERVICE_JOBS_NODE_REGEX = /(#{SERVICE_JOBS_PREFIXES})_node(.*)/i
+
+      SERVICE_AUXILIARY_JOBS = Set.new(%w(serialization_data_server backup_manager))
+
+      attr_reader :agent_id, :deployment, :job, :index, :metrics
+
+      def initialize(attributes = {})
+        super
+        @kind = :heartbeat
+        @metrics = []
+
+        @id = @attributes["id"]
+        @timestamp = Time.at(@attributes["timestamp"]) rescue @attributes["timestamp"]
+
+        @deployment = @attributes["deployment"]
+        @agent_id = @attributes["agent_id"]
+        @job = @attributes["job"]
+        @index = @attributes["index"].to_s
+        @job_state = @attributes["job_state"]
+
+        @tags = {}
+        @tags["job"] = @job if @job
+        @tags["index"] = @index if @index
+        @tags["role"] = guess_role
+
+        @vitals = @attributes["vitals"] || {}
+        @load = @vitals["load"] || []
+        @cpu = @vitals["cpu"] || {}
+        @mem = @vitals["mem"] || {}
+        @swap = @vitals["swap"] || {}
+        @disk = @vitals["disk"] || {}
+        @system_disk = @disk["system"] || {}
+        @ephemeral_disk = @disk["ephemeral"] || {}
+        @persistent_disk = @disk["persistent"] || {}
+
+        populate_metrics
+      end
+
+      def validate
+        add_error("id is missing") if @id.nil?
+        add_error("timestamp is missing") if @timestamp.nil?
+
+        if @timestamp && !@timestamp.kind_of?(Time)
+          add_error("timestamp is invalid")
+        end
+      end
+
+      def add_metric(name, value)
+        @metrics << Metric.new(name, value, @timestamp.to_i, @tags) if value
+      end
+
+      def short_description
+        "Heartbeat from #{@job}/#{@index} (#{@agent_id}) @ #{@timestamp.utc}"
+      end
+
+      def to_s
+        self.short_description
+      end
+
+      def to_hash
+        {
+          :kind => "heartbeat",
+          :id => @id,
+          :timestamp => @timestamp.to_i,
+          :deployment => @deployment,
+          :agent_id => @agent_id,
+          :job => @job,
+          :index => @index,
+          :job_state => @job_state,
+          :vitals => @vitals
+        }
+      end
+
+      def to_json
+        Yajl::Encoder.encode(self.to_hash)
+      end
+
+      def to_plain_text
+        self.short_description
+      end
+
+      private
+
+      def populate_metrics
+        add_metric("system.load.1m", @load[0]) if @load.kind_of?(Array)
+        add_metric("system.cpu.user", @cpu["user"])
+        add_metric("system.cpu.sys", @cpu["sys"])
+        add_metric("system.cpu.wait", @cpu["wait"])
+        add_metric("system.mem.percent", @mem["percent"])
+        add_metric("system.mem.kb", @mem["kb"])
+        add_metric("system.swap.percent", @swap["percent"])
+        add_metric("system.swap.kb", @swap["kb"])
+        add_metric("system.disk.system.percent", @system_disk["percent"])
+        add_metric("system.disk.system.inode_percent", @system_disk["inode_percent"])
+        add_metric("system.disk.ephemeral.percent", @ephemeral_disk["percent"])
+        add_metric("system.disk.ephemeral.inode_percent", @ephemeral_disk["inode_percent"])
+        add_metric("system.disk.persistent.percent", @persistent_disk["percent"])
+        add_metric("system.disk.persistent.inode_percent", @persistent_disk["inode_percent"])
+        add_metric("system.healthy", @job_state == "running" ? 1 : 0)
+      end
+
+      def guess_role
+        # Dashboard might want to partition jobs
+        # into several buckets, so let's help it
+        # by applying a couple of heuristics
+
+        return "core" if CORE_JOBS.include?(@job.to_s.downcase)
+
+        return "service" if SERVICE_AUXILIARY_JOBS.include?(@job.to_s.downcase)
+
+        # job name prefixed by "service"
+        if @job.to_s.downcase =~ /^service/i
+          return "service"
+        end
+
+        # job name suffixed by "_gateway"
+        if @job.to_s.downcase =~ SERVICE_JOBS_GATEWAY_REGEX
+          return "service"
+        end
+
+        # job name contains "_node"
+        if @job.to_s.downcase =~ SERVICE_JOBS_NODE_REGEX
+          return "service"
+        end
+
+        return "unknown"
+      end
+
+    end
+  end
+end

data/lib/bosh/monitor/metric.rb

@@ -0,0 +1,16 @@
+module Bosh::Monitor
+  class Metric
+
+    attr_accessor :name
+    attr_accessor :value
+    attr_accessor :timestamp
+    attr_accessor :tags
+
+    def initialize(name, value, timestamp, tags)
+      @name = name
+      @value = value
+      @timestamp = timestamp
+      @tags = tags
+    end
+  end
+end

data/lib/bosh/monitor/plugins/base.rb

@@ -0,0 +1,27 @@
+module Bosh::Monitor
+  module Plugins
+    class Base
+      attr_reader :logger
+      attr_reader :options
+      attr_reader :event_kinds
+
+      def initialize(options = {})
+        @logger  = Bhm.logger
+        @options = (options || {}).dup
+        @event_kinds = []
+      end
+
+      def validate_options
+        true
+      end
+
+      def run
+        raise FatalError, "`run' method is not implemented in `#{self.class}'"
+      end
+
+      def process(event)
+        raise FatalError, "`process' method is not implemented in `#{self.class}'"
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/cloud_watch.rb

@@ -0,0 +1,56 @@
+require 'aws-sdk'
+
+module Bosh::Monitor
+  module Plugins
+    class CloudWatch < Base
+      def initialize(options={})
+        @options = options
+      end
+
+      def aws_cloud_watch
+        @aws_cloud_watch ||= AWS::CloudWatch.new(@options)
+      end
+
+      def run
+      end
+
+      def process(event)
+        if event.is_a? Bosh::Monitor::Events::Heartbeat
+          aws_cloud_watch.put_metric_data(heartbeat_to_cloudwatch_metric(event))
+        end
+      end
+
+      private
+
+      def heartbeat_to_cloudwatch_metric(heartbeat)
+        {
+            namespace: "BOSH/HealthMonitor",
+            metric_data: heartbeat.metrics.collect do |metric|
+              build_metric(metric, dimensions(heartbeat))
+            end
+        }
+      end
+
+      def dimensions(heartbeat)
+        @dimensions ||= [
+            {name: "job", value: heartbeat.job},
+            {name: "index", value: heartbeat.index},
+            {name: "name", value: "#{heartbeat.job}/#{heartbeat.index}"},
+            {name: "deployment", value: heartbeat.deployment},
+            {name: "agent_id", value: heartbeat.agent_id}
+        ]
+      end
+
+      def build_metric(metric, dimensions)
+        timestamp = Time.at(metric.timestamp).utc.iso8601
+
+        {
+            metric_name: metric.name.to_s,
+            value: metric.value.to_s,
+            timestamp: timestamp,
+            dimensions: dimensions
+        }
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/datadog.rb

@@ -0,0 +1,78 @@
+require 'dogapi'
+
+module Bosh::Monitor
+  module Plugins
+    class DataDog < Base
+
+      NORMAL_PRIORITY = [:alert, :critical, :error]
+
+      def validate_options
+        !!(options.kind_of?(Hash) && options["api_key"] && options["application_key"])
+      end
+
+      def run
+        @api_key = options["api_key"]
+        @application_key = options["application_key"]
+        @pagerduty_service_name = options["pagerduty_service_name"]
+
+        logger.info("DataDog plugin is running...")
+      end
+
+      def dog_client
+        return @dog_client if @dog_client
+        client = Dogapi::Client.new(@api_key, @application_key)
+        @dog_client = @pagerduty_service_name ? PagingDatadogClient.new(@pagerduty_service_name, client) : client
+      end
+
+      def process(event)
+        case event
+          when Bosh::Monitor::Events::Heartbeat
+            EM.defer { process_heartbeat(event) }
+          when Bosh::Monitor::Events::Alert
+            EM.defer { process_alert(event) }
+          else
+            #ignore
+        end
+      end
+
+      private
+
+      def process_heartbeat(heartbeat)
+        tags = %W[
+          job:#{heartbeat.job}
+          index:#{heartbeat.index}
+          deployment:#{heartbeat.deployment}
+          agent:#{heartbeat.agent_id}
+        ]
+
+        heartbeat.metrics.each do |metric|
+          point = [Time.at(metric.timestamp), metric.value]
+          dog_client.emit_points("bosh.healthmonitor.#{metric.name}", [point], tags: tags)
+        end
+      end
+
+      def process_alert(alert)
+        msg, title, source, timestamp = alert.to_hash.values_at(:summary,
+                                                                :title,
+                                                                :source,
+                                                                :created_at)
+
+
+        # DataDog only supports "low" and "normal" priority
+        priority = normal_priority?(alert.severity) ? "normal" : "low"
+        dog_client.emit_event(
+          Dogapi::Event.new(msg,
+            msg_title: title,
+            date_happened: timestamp,
+            tags: ["source:#{source}"],
+            priority: priority
+          )
+        )
+      end
+
+      def normal_priority?(severity)
+        NORMAL_PRIORITY.include?(severity)
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/dummy.rb

@@ -0,0 +1,20 @@
+module Bosh::Monitor
+  module Plugins
+    class Dummy < Base
+      def run
+        logger.info("Dummy delivery agent is running...")
+      end
+
+      def process(event)
+        logger.info("Processing event!")
+        logger.info(event)
+        @events ||= []
+        @events << event
+      end
+
+      def events
+        @events
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/email.rb

@@ -0,0 +1,135 @@
+module Bosh::Monitor
+  module Plugins
+    class Email < Base
+      DEFAULT_INTERVAL = 10
+
+      def initialize(options = {})
+        @queues = {}
+        @lock  = Mutex.new
+
+        if options.has_key?("interval")
+          @delivery_interval = options["interval"].to_f
+        else
+          @delivery_interval = DEFAULT_INTERVAL
+        end
+
+        @started = false
+        super
+      end
+
+      def queue_size(kind)
+        return 0 if @queues[kind].nil?
+        @queues[kind].size
+      end
+
+      def run
+        unless EM.reactor_running?
+          logger.error("Email plugin can only be started when event loop is running")
+          return false
+        end
+
+        return true if @started
+        logger.info("Email plugin is running...")
+
+        EM.add_periodic_timer(@delivery_interval) do
+          begin
+            process_queues
+          rescue => e
+            logger.error("Problem processing email queues: #{e}")
+          end
+        end
+        @started = true
+      end
+
+      def validate_options
+        options.kind_of?(Hash) &&
+          options["recipients"].kind_of?(Array) &&
+          options["smtp"].kind_of?(Hash) &&
+          options["smtp"]["host"] &&
+          options["smtp"]["port"] &&
+          options["smtp"]["from"] &&
+          true # force the whole method to return Boolean
+      end
+
+      def recipients
+        options["recipients"]
+      end
+
+      def smtp_options
+        options["smtp"]
+      end
+
+      def process(event)
+        @lock.synchronize do
+          @queues[event.kind] ||= []
+          @queues[event.kind] << event
+        end
+      end
+
+      def process_queues
+        @queues.each_pair do |kind, queue|
+          next if queue.empty?
+          email_subject = "%s from BOSH Health Monitor" % [ pluralize(queue_size(kind), kind) ]
+          email_body = ""
+
+          @lock.synchronize do
+            while event = queue.shift
+              email_body << event.to_plain_text << "\n"
+            end
+          end
+
+          send_email_async(email_subject, email_body)
+        end
+      end
+
+      def send_email_async(subject, body, date = Time.now)
+        started = Time.now
+        logger.debug("Sending email...")
+
+        headers = {
+          "From"         => smtp_options["from"],
+          "To"           => recipients.join(", "),
+          "Subject"      => subject,
+          "Date"         => date,
+          "Content-Type" => "text/plain; charset=\"iso-8859-1\""
+        }
+
+        smtp_client_options = {
+          :domain   => smtp_options["domain"],
+          :host     => smtp_options["host"],
+          :port     => smtp_options["port"],
+          :from     => smtp_options["from"],
+          :to       => recipients,
+          :header   => headers,
+          :body     => body
+        }
+
+        if smtp_options["tls"]
+          smtp_client_options[:starttls] = true
+        end
+
+        if smtp_options["auth"]
+          smtp_client_options[:auth] = {
+            # FIXME: EM SMTP client will only work with plain auth
+            :type     => smtp_options["auth"].to_sym,
+            :username => smtp_options["user"],
+            :password => smtp_options["password"]
+          }
+        end
+
+        email = EM::Protocols::SmtpClient.send(smtp_client_options)
+
+        email.callback do
+          logger.debug("Email sent (took #{Time.now - started} seconds)")
+        end
+
+        email.errback do |e|
+          logger.error("Failed to send email: #{e}")
+        end
+
+      rescue => e
+        logger.error("Error sending email: #{e}")
+      end
+    end
+  end
+end