bosh-monitor 1.5.0.pre.1113

data/lib/bosh/monitor/runner.rb

@@ -0,0 +1,162 @@
+module Bosh::Monitor
+  class Runner
+    include YamlHelper
+
+    def self.run(config_file)
+      new(config_file).run
+    end
+
+    def initialize(config_file)
+      Bhm.config = load_yaml_file(config_file)
+
+      @logger        = Bhm.logger
+      @director      = Bhm.director
+      @intervals     = Bhm.intervals
+      @mbus          = Bhm.mbus
+      @agent_manager = Bhm.agent_manager
+    end
+
+    def run
+      @logger.info("HealthMonitor starting...")
+      EM.kqueue if EM.kqueue?
+      EM.epoll if EM.epoll?
+
+      EM.error_handler { |e| handle_em_error(e) }
+
+      EM.run do
+        connect_to_mbus
+        @director_monitor = DirectorMonitor.new(Bhm)
+        @director_monitor.subscribe
+        @agent_manager.setup_events
+        setup_timers
+        start_http_server
+        @logger.info "BOSH HealthMonitor #{Bhm::VERSION} is running..."
+      end
+    end
+
+    def stop(soft=false)
+      @logger.info("HealthMonitor shutting down...")
+      @http_server.stop! if @http_server
+      EM.stop
+      exit(0) unless soft
+    end
+
+    def setup_timers
+      EM.next_tick do
+        poll_director
+        EM.add_periodic_timer(@intervals.poll_director) { poll_director }
+        EM.add_periodic_timer(@intervals.log_stats) { log_stats }
+
+        EM.add_timer(@intervals.poll_grace_period) do
+          EM.add_periodic_timer(@intervals.analyze_agents) { analyze_agents }
+        end
+      end
+    end
+
+    def log_stats
+      n_deployments = pluralize(@agent_manager.deployments_count, "deployment")
+      n_agents = pluralize(@agent_manager.agents_count, "agent")
+      @logger.info("Managing #{n_deployments}, #{n_agents}")
+      @logger.info("Agent heartbeats received = %s" % [ @agent_manager.heartbeats_received ])
+    end
+
+    def connect_to_mbus
+      NATS.on_error do |e|
+        unless @shutting_down
+          if e.kind_of?(NATS::ConnectError)
+            handle_em_error(e)
+          else
+            log_exception(e)
+          end
+        end
+      end
+
+      nats_client_options = {
+        :uri       => @mbus.endpoint,
+        :user      => @mbus.user,
+        :pass      => @mbus.password,
+        :autostart => false
+      }
+
+      Bhm.nats = NATS.connect(nats_client_options) do
+        @logger.info("Connected to NATS at `#{@mbus.endpoint}'")
+      end
+    end
+
+    def start_http_server
+      @logger.info "HTTP server is starting on port #{Bhm.http_port}..."
+      @http_server = Thin::Server.new("0.0.0.0", Bhm.http_port, :signals => false) do
+        Thin::Logging.silent = true
+        use Rack::Auth::Basic do |user, password|
+          [ user, password ] == [ Bhm.http_user, Bhm.http_password ]
+        end
+        map "/" do
+          run Bhm::ApiController.new
+        end
+      end
+      @http_server.start!
+    end
+
+    def poll_director
+      @logger.debug "Getting deployments from director..."
+      Fiber.new { fetch_deployments }.resume
+      Bhm.set_varz("deployments_count", @agent_manager.deployments_count)
+      Bhm.set_varz("agents_count", @agent_manager.agents_count)
+    end
+
+    def analyze_agents
+      # N.B. Yes, his will block event loop,
+      # possibly consider deferring
+      @agent_manager.analyze_agents
+    end
+
+    private
+
+    # This is somewhat controversial approach: instead of swallowing some exceptions
+    # and letting event loop run further we force our server to stop. The rationale
+    # behind that is to avoid the situation when swallowed exception actually breaks
+    # things:
+    # 1. Periodic timer will get canceled unless we manually reschedule it
+    #    in a rescue clause even if we swallow the exception.
+    # 2. If we want to perform an operation on next tick AND schedule some operation
+    #    to be run periodically AND there is an exception swallowed somewhere during the
+    #    event processing, then on the next tick we don't really process events that follow the buggy one.
+    # These things can be pretty painful for HM as we might think it runs fine
+    # when it actually just swallows some exception and effectively does nothing.
+    # We might revisit that later
+    def handle_em_error(e)
+      @shutting_down = true
+      log_exception(e, :fatal)
+      stop
+    end
+
+    def log_exception(e, level = :error)
+      level = :error unless level == :fatal
+      @logger.send(level, e.to_s)
+      if e.respond_to?(:backtrace) && e.backtrace.respond_to?(:join)
+        @logger.send(level, e.backtrace.join("\n"))
+      end
+    end
+
+    def fetch_deployments
+      deployments = @director.get_deployments
+
+      @agent_manager.sync_deployments(deployments)
+
+      deployments.each do |deployment|
+        deployment_name = deployment["name"]
+
+        @logger.info "Found deployment `#{deployment_name}'"
+
+        vms = @director.get_deployment_vms(deployment_name)
+        @logger.debug "Fetching VMs information for `#{deployment_name}'..."
+
+        @agent_manager.sync_agents(deployment_name, vms)
+      end
+
+    rescue Bhm::DirectorError => e
+      log_exception(e)
+    end
+
+  end
+end

data/lib/bosh/monitor/version.rb

@@ -0,0 +1,5 @@
+module Bosh
+  module Monitor
+    VERSION = '1.5.0.pre.1113'
+  end
+end

data/lib/bosh/monitor/yaml_helper.rb

@@ -0,0 +1,18 @@
+module Bosh::Monitor
+  module YamlHelper
+
+    def load_yaml_file(path, expected_type = Hash)
+      raise(ConfigError, "Cannot find file `#{path}'") unless File.exists?(path)
+      yaml = Psych.load_file(path)
+
+      if expected_type && !yaml.is_a?(expected_type)
+        raise ConfigError, "Incorrect file format in `#{path}', #{expected_type} expected"
+      end
+
+      yaml
+    rescue SystemCallError => e
+      raise ConfigError, "Cannot load YAML file at `#{path}': #{e}"
+    end
+
+  end
+end

metadata

@@ -0,0 +1,246 @@
+--- !ruby/object:Gem::Specification
+name: bosh-monitor
+version: !ruby/object:Gem::Version
+  version: 1.5.0.pre.1113
+  prerelease: 6
+platform: ruby
+authors:
+- VMware
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.10
+- !ruby/object:Gem::Dependency
+  name: logging
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: nats
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.28
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.28
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.5
+- !ruby/object:Gem::Dependency
+  name: dogapi
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: ! 'BOSH Health Monitor
+
+  cfd471'
+email: support@cloudfoundry.com
+executables:
+- bosh-monitor-console
+- bosh-monitor
+- listener
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/bosh/monitor.rb
+- lib/bosh/monitor/agent.rb
+- lib/bosh/monitor/agent_manager.rb
+- lib/bosh/monitor/api_controller.rb
+- lib/bosh/monitor/config.rb
+- lib/bosh/monitor/core_ext.rb
+- lib/bosh/monitor/director.rb
+- lib/bosh/monitor/director_monitor.rb
+- lib/bosh/monitor/errors.rb
+- lib/bosh/monitor/event_processor.rb
+- lib/bosh/monitor/events/alert.rb
+- lib/bosh/monitor/events/base.rb
+- lib/bosh/monitor/events/heartbeat.rb
+- lib/bosh/monitor/metric.rb
+- lib/bosh/monitor/plugins/base.rb
+- lib/bosh/monitor/plugins/cloud_watch.rb
+- lib/bosh/monitor/plugins/datadog.rb
+- lib/bosh/monitor/plugins/dummy.rb
+- lib/bosh/monitor/plugins/email.rb
+- lib/bosh/monitor/plugins/http_request_helper.rb
+- lib/bosh/monitor/plugins/logger.rb
+- lib/bosh/monitor/plugins/nats.rb
+- lib/bosh/monitor/plugins/pagerduty.rb
+- lib/bosh/monitor/plugins/paging_datadog_client.rb
+- lib/bosh/monitor/plugins/resurrector.rb
+- lib/bosh/monitor/plugins/resurrector_helper.rb
+- lib/bosh/monitor/plugins/tsdb.rb
+- lib/bosh/monitor/plugins/varz.rb
+- lib/bosh/monitor/protocols/tsdb.rb
+- lib/bosh/monitor/runner.rb
+- lib/bosh/monitor/version.rb
+- lib/bosh/monitor/yaml_helper.rb
+- README
+- bin/bosh-monitor-console
+- bin/bosh-monitor
+- bin/listener
+homepage: https://github.com/cloudfoundry/bosh
+licenses:
+- Apache 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: BOSH Health Monitor
+test_files: []