bosh-monitor 1.5.0.pre.1113

data/lib/bosh/monitor/plugins/http_request_helper.rb

@@ -0,0 +1,25 @@
+module Bosh::Monitor::Plugins
+  module HttpRequestHelper
+    def send_http_post_request(uri, request)
+      send_http_request(:post, uri, request)
+    end
+
+    def send_http_put_request(uri, request)
+      send_http_request(:put, uri, request)
+    end
+
+    def send_http_request(method, uri, request)
+      name = self.class.name
+      logger.debug("sending HTTP #{method.to_s.upcase} to: #{uri}")
+      started = Time.now
+      http = EM::HttpRequest.new(uri).send(method, request)
+      http.callback do
+        logger.debug("#{name} event sent (took #{Time.now - started} seconds): #{http.response_header.status}")
+      end
+
+      http.errback do |e|
+        logger.error("Failed to send #{name} event: #{e.error}")
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/logger.rb

@@ -0,0 +1,13 @@
+module Bosh::Monitor
+  module Plugins
+    class Logger < Base
+      def run
+        logger.info("Logging delivery agent is running...")
+      end
+
+      def process(event)
+        logger.info("[#{event.kind.to_s.upcase}] #{event}")
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/nats.rb

@@ -0,0 +1,43 @@
+module Bosh::Monitor
+  module Plugins
+    class Nats < Base
+      SUBJECT = "bosh.hm.events"
+
+      def validate_options
+        options.kind_of?(Hash) &&
+          options["endpoint"] &&
+          options.has_key?("user") &&
+          options.has_key?("password")
+      end
+
+      def run
+        unless EM.reactor_running?
+          logger.error("NATS delivery agent can only be started when event loop is running")
+          return false
+        end
+
+        nats_client_options = {
+          :uri       => options["endpoint"],
+          :user      => options["user"],
+          :pass      => options["password"],
+          :autostart => false
+        }
+
+        @nats = NATS.connect(nats_client_options) do
+          logger.info("Ready to publish alerts to NATS at `#{options["endpoint"]}'")
+        end
+      end
+
+      def process(event)
+        if @nats.nil?
+          @logger.error("Cannot deliver event, NATS not initialized")
+          return false
+        end
+
+        nats_subject = options["subject"] || SUBJECT
+        @nats.publish(nats_subject, event.to_json)
+        true
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/pagerduty.rb

@@ -0,0 +1,48 @@
+module Bosh::Monitor
+  module Plugins
+    class Pagerduty < Base
+      include Bosh::Monitor::Plugins::HttpRequestHelper
+
+      API_URI = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
+
+      def run
+        unless EM.reactor_running?
+          logger.error("Pagerduty plugin can only be started when event loop is running")
+          return false
+        end
+
+        logger.info("Pagerduty delivery agent is running...")
+      end
+
+      def validate_options
+        options.kind_of?(Hash) &&
+          options["service_key"].kind_of?(String)
+      end
+
+      def process(event)
+        started = Time.now
+
+        payload = {
+          :service_key  => options["service_key"],
+          :event_type   => "trigger",
+          :incident_key => event.id,
+          :description  => event.short_description,
+          :details      => event.to_hash
+        }
+
+        request = {
+          :body => Yajl::Encoder.encode(payload)
+        }
+
+        if options["http_proxy"]
+          proxy = URI.parse(options["http_proxy"])
+          request[:proxy] = { :host => proxy.host, :port => proxy.port }
+        end
+
+        send_http_post_request(API_URI, request)
+      rescue => e
+        logger.error("Error sending pagerduty event: #{e}")
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/paging_datadog_client.rb

@@ -0,0 +1,24 @@
+class PagingDatadogClient
+  attr_reader :datadog_recipient
+
+  def initialize(datadog_recipient, datadog_client)
+    @datadog_recipient = datadog_recipient
+    @datadog_client = datadog_client
+  end
+
+  def emit_points(metric, points, options={})
+    @datadog_client.emit_points(metric, points, options)
+  end
+
+  def emit_event(event)
+    event_hash = event.to_hash
+    new_message = if event.priority == "normal"
+                    "#{event.msg_text} @#{@datadog_recipient}"
+                  else
+                    event.msg_text
+                  end
+    new_event = Dogapi::Event.new(new_message, event_hash)
+
+    @datadog_client.emit_event(new_event)
+  end
+end

data/lib/bosh/monitor/plugins/resurrector.rb

@@ -0,0 +1,82 @@
+# This health monitor plugin should be used in conjunction with another plugin that
+# alerts when a VM is unresponsive, as this plugin will try to automatically fix the
+# problem by recreating the VM
+module Bosh::Monitor
+  module Plugins
+    class Resurrector < Base
+      include Bosh::Monitor::Plugins::HttpRequestHelper
+
+      attr_reader :url
+
+      def initialize(options={})
+        super(options)
+        director = @options['director']
+        raise ArgumentError 'director options not set' unless director
+
+        @url           = URI(director['endpoint'])
+        @user          = director['user']
+        @password      = director['password']
+        @processor     = Bhm.event_processor
+        @alert_tracker = ResurrectorHelper::AlertTracker.new(@options)
+      end
+
+      def run
+        unless EM.reactor_running?
+          logger.error("Resurrector plugin can only be started when event loop is running")
+          return false
+        end
+
+        logger.info("Resurrector is running...")
+      end
+
+      def process(alert)
+        deployment = alert.attributes['deployment']
+        job = alert.attributes['job']
+        index = alert.attributes['index']
+
+        # only when the agent times out do we add deployment, job & index to the alert
+        # attributes, so this won't trigger a recreate for other types of alerts
+        if deployment && job && index
+          agent_key = ResurrectorHelper::JobInstanceKey.new(deployment, job, index)
+          @alert_tracker.record(agent_key, alert.created_at)
+
+          payload = {'jobs' => {job => [index]}}
+          request = {
+              head: {
+                  'Content-Type' => 'application/json',
+                  'authorization' => [@user, @password]
+              },
+              body: Yajl::Encoder.encode(payload)
+          }
+
+          @url.path = "/deployments/#{deployment}/scan_and_fix"
+
+          if @alert_tracker.melting_down?(deployment)
+            # freak out
+            ts = Time.now.to_i
+            @processor.process(:alert,
+                               severity: 1,
+                               source: "HM plugin resurrector",
+                               title: "We are in meltdown.",
+                               created_at: ts)
+
+            logger.error("(Resurrector) we are in meltdown.")
+          else
+            # queue instead, and only queue if it isn't already in the queue
+            # what if we can't keep up with the failure rate?
+            # - maybe not, maybe the meltdown detection takes care of the rate issue
+            logger.warn("(Resurrector) notifying director to recreate unresponsive VM: #{deployment} #{job}/#{index}")
+
+            send_http_put_request(url.to_s, request)
+          end
+
+
+        else
+          logger.warn("(Resurrector) event did not have deployment, job and index: #{alert}")
+        end
+      end
+
+    end
+  end
+end
+

data/lib/bosh/monitor/plugins/resurrector_helper.rb

@@ -0,0 +1,84 @@
+module Bosh::Monitor::Plugins
+  module ResurrectorHelper
+
+    # Hashable tuple of the identifying properties of a job
+    class JobInstanceKey
+      attr_accessor :deployment, :job, :index
+
+      def initialize(deployment, job, index)
+        @deployment = deployment
+        @job        = job
+        @index      = index
+      end
+
+      def hash
+        (deployment.to_s + job.to_s + index.to_s).hash
+      end
+
+      def eql?(other)
+        other.deployment == deployment &&
+            other.job == job &&
+            other.index == index
+      end
+
+      def to_s
+        [deployment, job, index].join('/')
+      end
+    end
+
+    # Service which tracks alerts and decides whether or not the cluster is melting down.
+    # When the cluster is melting down, the resurrector backs off on fixing instances.
+    class AlertTracker
+
+      # Below this number of down agents we don't consider a meltdown occurring
+      attr_accessor :minimum_down_jobs
+
+      # Number of seconds at which an alert is considered "current"; alerts older than
+      # this are ignored. Integer number of seconds.
+      attr_accessor :time_threshold
+
+      # Percentage of the cluster which must be down for scanning to stop. Float fraction
+      # between 0 and 1.
+      attr_accessor :percent_threshold
+
+      def initialize(args={})
+        @agent_manager       = Bhm.agent_manager
+        @alert_times         = {} # maps JobInstanceKey to time of last Alert
+        @minimum_down_jobs   = args.fetch('minimum_down_jobs', 5)
+        @percent_threshold   = args.fetch('percent_threshold', 0.2)
+        @time_threshold      = args.fetch('time_threshold', 600)
+      end
+
+      # "Melting down" means a large part of the cluster is offline and manual intervention
+      # may be required to fix.
+      def melting_down?(deployment)
+        agent_alerts = alerts_for_deployment(deployment)
+        total_number_of_agents = agent_alerts.size
+        number_of_down_agents = agent_alerts.select { |_, alert_time|
+          alert_time > (Time.now - time_threshold)
+        }.size
+
+        return false if number_of_down_agents < minimum_down_jobs
+
+        (number_of_down_agents.to_f / total_number_of_agents) >= percent_threshold
+      end
+
+      def record(agent_key, alert_time)
+        @alert_times[agent_key] = alert_time
+      end
+
+      private
+
+      def alerts_for_deployment(deployment)
+        agents = @agent_manager.get_agents_for_deployment(deployment)
+        keys = agents.values.map { |agent|
+          JobInstanceKey.new(agent.deployment, agent.job, agent.index)
+        }
+
+        result = {}
+        keys.each { |key| result[key] = @alert_times.fetch(key, Time.at(0)) }
+        result
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/tsdb.rb

@@ -0,0 +1,43 @@
+module Bosh::Monitor
+  module Plugins
+    class Tsdb < Base
+
+      def validate_options
+        options.kind_of?(Hash) &&
+          options["host"] &&
+          options["port"] &&
+          true
+      end
+
+      def run
+        unless EM.reactor_running?
+          logger.error("TSDB delivery agent can only be started when event loop is running")
+          return false
+        end
+
+        host = options["host"]
+        port = options["port"]
+        @tsdb = EM.connect(host, port, Bhm::TsdbConnection, host, port)
+      end
+
+      def process(event)
+        if @tsdb.nil?
+          @logger.error("Cannot deliver event, TSDB connection is not initialized")
+          return false
+        end
+
+        metrics = event.metrics
+
+        if !metrics.kind_of?(Enumerable)
+          raise PluginError, "Invalid event metrics: Enumerable expected, #{metrics.class} given"
+        end
+
+        metrics.each do |metric|
+          @tsdb.send_metric(metric.name, metric.timestamp, metric.value, metric.tags)
+        end
+
+        true
+      end
+    end
+  end
+end

data/lib/bosh/monitor/plugins/varz.rb

@@ -0,0 +1,17 @@
+module Bosh::Monitor
+  module Plugins
+    class Varz < Base
+      def run
+        logger.info("Varz plugin is running...")
+      end
+
+      def process(event)
+        @agents ||= {}
+        @agents[event.kind] ||= {}
+        agent_id = event.attributes["agent_id"] || "unknown"
+        @agents[event.kind][agent_id.to_s] = event.to_hash
+        Bhm.set_varz("last_agents_" + event.kind.to_s, @agents[event.kind])
+      end
+    end
+  end
+end

data/lib/bosh/monitor/protocols/tsdb.rb

@@ -0,0 +1,68 @@
+module Bosh::Monitor
+  class TsdbConnection < EventMachine::Connection
+
+    BACKOFF_CEILING = 9
+    MAX_RETRIES = 35
+
+    attr_reader :retries
+
+    def initialize(host, port)
+      @host = host
+      @port = port
+      @logger = Bhm.logger
+      reset_retries
+    end
+
+    def reset_retries
+      @retries = 0
+    end
+
+    def increment_retries
+      @retries += 1
+    end
+
+    def send_metric(name, timestamp, value, tags = {})
+      formatted_tags = tags.map { |tag| tag.join("=") }.sort.join(" ")
+      command = "put #{name} #{timestamp} #{value} #{formatted_tags}\n"
+      @logger.debug("[TSDB] >> #{command.chomp}")
+      send_data(command)
+    end
+
+    def connection_completed
+      reset_retries
+      @reconnecting = false
+      @connected = true
+      @logger.info("Connected to TSDB server at #{@host}:#{@port}")
+    end
+
+    def unbind
+      if @connected
+        @logger.warn("Lost connection to TSDB server at #{@host}:#{@port}")
+      end
+      @connected = false
+
+      retry_in = 2**[retries, BACKOFF_CEILING].min - 1
+      increment_retries
+
+      if retries > MAX_RETRIES
+        raise "Failed to reconnect to TSDB after #{MAX_RETRIES} retries"
+      end
+
+      if retries > 1
+        @logger.info("Failed to reconnect to TSDB, will try again in #{retry_in} seconds...")
+      end
+
+      EM.add_timer(retry_in) { tsdb_reconnect }
+    end
+
+    def tsdb_reconnect
+      @logger.info("Trying to reconnect to TSDB server at #{@host}:#{@port} (#{retries})...")
+      reconnect(@host, @port)
+    end
+
+    def receive_data(data)
+      @logger.info("[TSDB] << #{data.chomp}")
+    end
+
+  end
+end