bosh-monitor 1.5.0.pre.1113

data/lib/bosh/monitor/api_controller.rb

@@ -0,0 +1,18 @@
+module Bosh::Monitor
+
+  class ApiController < Sinatra::Base
+
+    configure do
+      set(:show_exceptions, false)
+      set(:raise_errors, false)
+      set(:dump_errors, false)
+    end
+
+    get "/varz" do
+      content_type(:json)
+      Yajl::Encoder.encode(Bhm.varz, :terminator => "\n")
+    end
+
+  end
+
+end

data/lib/bosh/monitor/config.rb

@@ -0,0 +1,71 @@
+module Bosh::Monitor
+
+  class << self
+
+    attr_accessor :logger
+    attr_accessor :director
+    attr_accessor :intervals
+    attr_accessor :mbus
+    attr_accessor :event_mbus
+    attr_accessor :agent_manager
+    attr_accessor :event_processor
+
+    attr_accessor :http_port, :http_user, :http_password
+    attr_accessor :plugins
+    attr_accessor :varz
+
+    attr_accessor :nats
+
+    def config=(config)
+      validate_config(config)
+
+      @logger = Logging.logger(config["logfile"] || STDOUT)
+      @intervals = OpenStruct.new(config["intervals"])
+      @director = Director.new(config["director"])
+      @mbus = OpenStruct.new(config["mbus"])
+
+      @event_processor = EventProcessor.new
+      @agent_manager = AgentManager.new(event_processor)
+
+      @varz = {}
+
+      # Interval defaults
+      @intervals.prune_events ||= 30
+      @intervals.poll_director ||= 60
+      @intervals.poll_grace_period ||= 30
+      @intervals.log_stats ||= 60
+      @intervals.analyze_agents ||= 60
+      @intervals.agent_timeout ||= 60
+      @intervals.rogue_agent_alert ||= 120
+
+      if config["http"].is_a?(Hash)
+        @http_port = config["http"]["port"]
+        @http_user = config["http"]["user"]
+        @http_password = config["http"]["password"]
+      end
+
+      if config["event_mbus"]
+        @event_mbus = OpenStruct.new(config["event_mbus"])
+      end
+
+      if config["loglevel"].is_a?(String)
+        @logger.level = config["loglevel"].to_sym
+      end
+
+      if config["plugins"].is_a?(Enumerable)
+        @plugins = config["plugins"]
+      end
+    end
+
+    def set_varz(key, value)
+      @varz ||= {}
+      @varz[key] = value
+    end
+
+    def validate_config(config)
+      unless config.is_a?(Hash)
+        raise ConfigError, "Invalid config format, Hash expected, #{config.class} given"
+      end
+    end
+  end
+end

data/lib/bosh/monitor/core_ext.rb

@@ -0,0 +1,8 @@
+module Kernel
+
+  def pluralize(number, singular, plural = nil)
+    plural = plural || "#{singular}s"
+    number == 1 ? "1 #{singular}" : "#{number} #{plural}"
+  end
+
+end

data/lib/bosh/monitor/director.rb

@@ -0,0 +1,76 @@
+module Bosh::Monitor
+  class Director
+
+    def initialize(options)
+      @endpoint = options["endpoint"].to_s
+      @user     = options["user"].to_s
+      @password = options["password"].to_s
+    end
+
+    def get_deployments
+      http = perform_request(:get, "/deployments")
+
+      body   = http.response
+      status = http.response_header.http_status
+
+      if status != "200"
+        raise DirectorError, "Cannot get deployments from director at #{http.uri}: #{status} #{body}"
+      end
+
+      parse_json(body, Array)
+    end
+
+    def get_deployment_vms(name)
+      http = perform_request(:get, "/deployments/#{name}/vms")
+
+      body   = http.response
+      status = http.response_header.http_status
+
+      if status != "200"
+        raise DirectorError, "Cannot get deployment `#{name}' from director at #{http.uri}: #{status} #{body}"
+      end
+
+      parse_json(body, Array)
+    end
+
+    private
+
+    def parse_json(json, expected_type = nil)
+      result = Yajl::Parser.parse(json)
+
+      if expected_type && !result.kind_of?(expected_type)
+        raise DirectorError, "Invalid JSON response format, expected #{expected_type}, got #{result.class}"
+      end
+
+      result
+
+    rescue Yajl::ParseError => e
+      raise DirectorError, "Cannot parse director response: #{e.message}"
+    end
+
+    # JMS and GO: This effectively turns async requests into synchronous requests.
+    # This is a very bad thing to do on eventmachine because it will block the single
+    # event loop. This code should be removed and all requests converted
+    # to "the eventmachine way".
+    def perform_request(method, uri)
+      f = Fiber.current
+
+      target_uri = @endpoint + uri
+
+      headers = {
+        "authorization" => [@user, @password]
+      }
+
+      http = EM::HttpRequest.new(target_uri).send(method.to_sym, :head => headers)
+
+      http.callback { f.resume(http) }
+      http.errback  { f.resume(http) }
+
+      Fiber.yield
+
+    rescue URI::Error => e
+      raise DirectorError, "Invalid URI: #{target_uri}"
+    end
+
+  end
+end

data/lib/bosh/monitor/director_monitor.rb

@@ -0,0 +1,33 @@
+module Bosh::Monitor
+  class DirectorMonitor
+    def initialize(config)
+      @nats = config.nats
+      @logger = config.logger
+      @event_processor = config.event_processor
+    end
+
+    def subscribe
+      @nats.subscribe('hm.director.alert') do |msg|
+        alert = Yajl::Parser.parse(msg)
+
+        if valid_payload?(alert)
+          @event_processor.process(:alert, alert)
+        end
+      end
+    end
+
+    private
+
+    def valid_payload?(payload)
+      missing_keys = %w(id severity title summary created_at) - payload.keys
+      valid = missing_keys.empty?
+
+      unless valid
+        first_missing_key = missing_keys.first
+        @logger.error("Invalid payload from director: the key '#{first_missing_key}' was missing. #{payload.inspect}")
+      end
+
+      valid
+    end
+  end
+end

data/lib/bosh/monitor/errors.rb

@@ -0,0 +1,19 @@
+module Bosh::Monitor
+
+  class Error < StandardError
+    def self.code(code = nil)
+      define_method(:code) { code }
+    end
+  end
+
+  class FatalError < Error; code(42); end
+
+  class ConfigError < Error; code(101); end
+  class DirectorError < Error; code(201); end
+  class ConnectionError < Error; code(202); end
+
+  class EventProcessingError < Error; code(301); end
+  class InvalidEvent < Error; code(302); end
+
+  class PluginError < Error; code(401); end
+end

data/lib/bosh/monitor/event_processor.rb

@@ -0,0 +1,109 @@
+module Bosh::Monitor
+  class EventProcessor
+    attr_reader :plugins
+
+    def initialize
+      @events = {}
+      @plugins = {}
+
+      @lock   = Mutex.new
+      @logger = Bhm.logger
+    end
+
+    def add_plugin(plugin, event_kinds = [])
+      if plugin.respond_to?(:validate_options) && !plugin.validate_options
+        raise FatalError, "Invalid plugin options for `#{plugin.class}'"
+      end
+
+      @lock.synchronize do
+        event_kinds.each do |kind|
+          kind = kind.to_sym
+          @plugins[kind] ||= Set.new
+          @plugins[kind] << plugin
+        end
+        plugin.run
+      end
+    end
+
+    def process(kind, data)
+      kind = kind.to_sym
+      event = Bhm::Events::Base.create!(kind, data)
+
+      @lock.synchronize do
+        @events[kind] ||= {}
+
+        if @events[kind].has_key?(event.id)
+          @logger.debug("Ignoring duplicate #{event.kind} `#{event.id}'")
+          return true
+        end
+        # We don't really need to store event itself for the moment,
+        # as we only use its id to dedup new events.
+        @events[kind][event.id] = { :received_at => Time.now.to_i }
+      end
+
+      if @plugins[kind].nil? || @plugins[kind].empty?
+        @logger.debug("No plugins are interested in `#{event.kind}' event")
+        return true
+      end
+
+      @plugins[kind].each do |plugin|
+        plugin_process(plugin, event)
+      end
+
+      true
+    end
+
+    def events_count
+      # Accumulate event counter over all event kinds
+      @lock.synchronize do
+        @events.inject(0) do |counter, (kind, events)|
+          counter += events.size
+        end
+      end
+    end
+
+    def enable_pruning(interval)
+      @reaper ||= Thread.new do
+        loop do
+          # Some events might be in the system up to 2 * interval
+          # seconds this way, but it seems to be a reasonable trade-off
+          prune_events(interval)
+          sleep(interval)
+        end
+      end
+    end
+
+    def prune_events(lifetime)
+      @lock.synchronize do
+        pruned_count = 0
+        total_count = 0
+
+        @events.each_value do |list|
+          list.delete_if do |id, data|
+            total_count += 1
+            if data[:received_at] <= Time.now.to_i - lifetime
+              pruned_count += 1
+              true
+            else
+              false
+            end
+          end
+        end
+
+        @logger.debug("Pruned %s" % [ pluralize(pruned_count, "old event") ])
+        @logger.debug("Total %s" % [ pluralize(total_count, "event") ])
+      end
+    rescue => e
+      @logger.error("Error pruning events: #{e}")
+      @logger.error(e.backtrace.join("\n"))
+    end
+
+    private
+
+    def plugin_process(plugin, event)
+      plugin.process(event)
+    rescue Bhm::PluginError => e
+      @logger.error("Plugin #{plugin.class} failed to process #{event.kind}: #{e}")
+    end
+  end
+end

data/lib/bosh/monitor/events/alert.rb

@@ -0,0 +1,92 @@
+module Bosh::Monitor
+  module Events
+    class Alert < Base
+
+      # Considering Bosh::Agent::Alert
+      SEVERITY_MAP = {
+        1 => :alert,
+        2 => :critical,
+        3 => :error,
+        4 => :warning,
+        -1 => :ignored
+      }
+
+      attr_reader :created_at, :source, :title
+
+      def initialize(attributes = {})
+        super
+        @kind = :alert
+
+        @id         = @attributes["id"]
+        @severity   = @attributes["severity"]
+        @title      = @attributes["title"]
+        @summary    = @attributes["summary"] || @title
+        @source     = @attributes["source"]
+
+        # This rescue is just to preserve existing test behavior. However, this
+        # seems like a pretty wacky way to handle errors - wouldn't we rather
+        # have a nice exception?
+        @created_at = Time.at(@attributes["created_at"]) rescue @attributes["created_at"]
+      end
+
+      def validate
+        add_error("id is missing") if @id.nil?
+        add_error("severity is missing") if @severity.nil?
+
+        if @severity && (!@severity.kind_of?(Integer) || @severity < 0)
+          add_error("severity is invalid (non-negative integer expected)")
+        end
+
+        add_error("title is missing") if @title.nil?
+        add_error("timestamp is missing") if @created_at.nil?
+
+        if @created_at && !@created_at.kind_of?(Time)
+          add_error('created_at is invalid UNIX timestamp')
+        end
+      end
+
+      def short_description
+        "Severity #{@severity}: #{@source} #{@title}"
+      end
+
+      def severity
+        SEVERITY_MAP[@severity] || @severity
+      end
+
+      def to_hash
+        {
+          :kind       => "alert",
+          :id         => @id,
+          :severity   => @severity,
+          :title      => @title,
+          :summary    => @summary,
+          :source     => @source,
+          :created_at => @created_at.to_i
+        }
+      end
+
+      def to_json
+        Yajl::Encoder.encode(self.to_hash)
+      end
+
+      def to_s
+        "Alert @ #{@created_at.utc}, severity #{@severity}: #{@summary}"
+      end
+
+      def to_plain_text
+        result = ""
+        result << "#{@source}\n" unless @source.nil?
+        result << (@title || "Unknown Alert") << "\n"
+        result << "Severity: #{@severity}\n"
+        result << "Summary: #{@summary}\n" unless @summary.nil?
+        result << "Time: #{@created_at.utc}\n"
+        result
+      end
+
+      def metrics
+        [ ]
+      end
+
+    end
+  end
+end

data/lib/bosh/monitor/events/base.rb

@@ -0,0 +1,70 @@
+module Bosh::Monitor
+  module Events
+    class Base
+      attr_accessor :id
+
+      attr_reader :logger
+      attr_reader :kind
+      attr_reader :attributes
+      attr_reader :errors
+
+      def self.create!(kind, attributes = {})
+        event = create(kind, attributes)
+        if !event.valid?
+          raise InvalidEvent, event.error_message
+        end
+        event
+      end
+
+      def self.create(kind, attributes = {})
+        if !attributes.kind_of?(Hash)
+          raise InvalidEvent, "Cannot create event from #{attributes.class}"
+        end
+
+        case kind.to_s
+        when "heartbeat"
+          klass = Bhm::Events::Heartbeat
+        when "alert"
+          klass = Bhm::Events::Alert
+        else
+          raise InvalidEvent, "Cannot find `#{kind}' event handler"
+        end
+
+        event = klass.new(attributes)
+        event.id = SecureRandom.uuid if event.id.nil?
+        event
+      end
+
+      def initialize(attributes = {})
+        @attributes = {}
+        @kind = :unknown
+
+        attributes.each_pair do |k, v|
+          @attributes[k.to_s] = v
+        end
+
+        @logger = Bhm.logger
+        @errors = Set.new
+      end
+
+      def add_error(error)
+        @errors << error
+      end
+
+      def valid?
+        validate
+        @errors.empty?
+      end
+
+      def error_message
+        @errors.to_a.join(", ")
+      end
+
+      [:validate, :to_plain_text, :to_hash, :to_json, :metrics].each do |method|
+        define_method(method) do
+          raise FatalError, "`#{method}' is not implemented by #{self.class}"
+        end
+      end
+    end
+  end
+end