bosh-director 1.5.0.pre.1113

data/lib/bosh/director/problem_handlers/unbound_instance_vm.rb

@@ -0,0 +1,85 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ProblemHandlers
+    class UnboundInstanceVm < Base
+
+      register_as :unbound_instance_vm
+      auto_resolution :reassociate_vm
+
+      def initialize(vm_id, data)
+        super
+
+        @vm = Models::Vm[vm_id]
+        @data = data
+
+        if @vm.nil?
+          handler_error("VM `#{vm_id}' is no longer in the database")
+        end
+
+        if @vm.agent_id.nil?
+          handler_error("VM `#{vm_id}' doesn't have an agent id")
+        end
+
+        if @vm.cid.nil?
+          handler_error("VM `#{vm_id}' doesn't have a cloud id")
+        end
+
+      end
+
+      def description
+        job = @data["job"] || "unknown job"
+        index = @data["index"] || "unknown index"
+        "VM `#{@vm.cid}' reports itself as `#{job}/#{index}' but does not have a bound instance"
+      end
+
+      resolution :ignore do
+        plan { "Ignore problem" }
+        action { }
+      end
+
+      resolution :delete_vm do
+        plan { "Delete VM (unless it has persistent disk)" }
+        action { validate; delete_vm(@vm) }
+      end
+
+      resolution :reassociate_vm do
+        plan { "Reassociate VM with corresponding instance" }
+        action { validate; reassociate_vm }
+      end
+
+      def validate
+        unless @vm.instance.nil?
+          handler_error("Instance is now bound to VM")
+        end
+
+        state = agent_timeout_guard(@vm) { |agent| agent.get_state }
+        if state["job"].nil?
+          handler_error("VM now properly reports no job")
+        end
+      end
+
+      def reassociate_vm
+        instances = Models::Instance.
+          filter(:deployment_id => @vm.deployment_id,
+                 :job => @data["job"], :index => @data["index"]).all
+
+        if instances.size > 1
+          handler_error("More than one instance in DB matches this VM")
+        end
+
+        if instances.empty?
+          handler_error("No instances in DB match this VM")
+        end
+
+        instance = instances[0]
+
+        if instance.vm
+          handler_error("The corresponding instance is associated with another VM")
+        end
+
+        instance.update(:vm => @vm)
+      end
+    end
+  end
+end

data/lib/bosh/director/problem_handlers/unresponsive_agent.rb

@@ -0,0 +1,78 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ProblemHandlers
+    class UnresponsiveAgent < Base
+
+      register_as :unresponsive_agent
+      auto_resolution :ignore
+
+      def initialize(vm_id, data)
+        super
+        @vm = Models::Vm[vm_id]
+
+        if @vm.nil?
+          handler_error("VM `#{vm_id}' is no longer in the database")
+        end
+
+        if @vm.agent_id.nil?
+          handler_error("VM `#{vm_id}' doesn't have an agent id")
+        end
+      end
+
+      def description
+        "#{instance_name(@vm)} (#{@vm.cid}) is not responding"
+      end
+
+      resolution :ignore do
+        plan { "Ignore problem" }
+        action { }
+      end
+
+      resolution :reboot_vm do
+        plan { "Reboot VM" }
+        action { validate; ensure_cid; reboot_vm(@vm) }
+      end
+
+      resolution :recreate_vm do
+        plan { "Recreate VM using last known apply spec" }
+        action { validate; ensure_cid; recreate_vm(@vm) }
+      end
+
+      resolution :delete_vm_reference do
+        plan { "Delete VM reference (DANGEROUS!)" }
+        action { validate; ensure_no_cid; delete_vm_reference(@vm) }
+      end
+
+      def agent_alive?
+        agent_client(@vm).ping
+        true
+      rescue Bosh::Director::RpcTimeout
+        false
+      end
+
+      def ensure_cid
+        if @vm.cid.nil?
+          handler_error("VM `#{@vm.id}' doesn't have a cloud id, " +
+                            "only resolution is to delete the VM reference.")
+        end
+      end
+
+      def ensure_no_cid
+        if @vm.cid
+          handler_error("VM `#{@vm.id}' has a cloud id, " +
+                            "please use a different resolution.")
+        end
+      end
+
+      def validate
+        if agent_alive?
+          handler_error("Agent is responding now, skipping resolution")
+        end
+      end
+
+      def delete_vm
+      end
+    end
+  end
+end

data/lib/bosh/director/problem_resolver.rb

@@ -0,0 +1,103 @@
+module Bosh::Director
+  class ProblemResolver
+
+    attr_reader :event_log, :logger
+
+    def initialize(deployment)
+      @deployment = deployment
+      @resolved_count = 0
+
+      #temp
+      @event_log = Config.event_log
+      @logger = Config.logger
+    end
+
+    def begin_stage(stage_name, n_steps)
+      event_log.begin_stage(stage_name, n_steps)
+      logger.info(stage_name)
+    end
+
+    def track_and_log(task, log = true)
+      event_log.track(task) do |ticker|
+        logger.info(task) if log
+        yield ticker if block_given?
+      end
+    end
+
+    def apply_resolutions(resolutions)
+      @resolutions = resolutions
+      problems = Models::DeploymentProblem.filter(deployment: @deployment, state: 'open').all
+      problem_ids = Set.new
+
+      problems.each do |problem|
+        problem_ids << problem.id.to_s
+        unless @resolutions.has_key?(problem.id.to_s)
+          raise CloudcheckResolutionNotProvided,
+                "Resolution for problem #{problem.id} (#{problem.type}) is not provided"
+        end
+      end
+
+      # We might have some resolutions for problems that are no longer open
+      # or just some bogus problem ids, in that case we still need to mention
+      # them in event log so end user understands what actually happened.
+      missing_problem_ids = @resolutions.keys.to_set - problem_ids
+
+      begin_stage("Applying problem resolutions", problems.size + missing_problem_ids.size)
+      problems.each do |problem|
+        apply_resolution(problem)
+      end
+
+      missing_problem_ids.each do |problem_id|
+        if problem_id !~ /^\d+$/
+          reason = "malformed id"
+        else
+          problem = Models::DeploymentProblem[problem_id.to_i]
+          if problem.nil?
+            reason = "not found"
+          elsif problem.state != "open"
+            reason = "state is '#{problem.state}'"
+          elsif problem.deployment_id != @deployment.id
+            reason = "not a part of this deployment"
+          else
+            reason = "reason unknown"
+          end
+        end
+
+        track_and_log("Ignoring problem #{problem_id} (#{reason})") { }
+      end
+      @resolved_count
+    end
+
+    def apply_resolution(problem)
+      handler = ProblemHandlers::Base.create_from_model(problem)
+      handler.job = self
+
+      resolution = @resolutions[problem.id.to_s] || handler.auto_resolution
+      problem_summary = "#{problem.type} #{problem.resource_id}"
+      resolution_summary = handler.resolution_plan(resolution)
+      resolution_summary ||= "no resolution"
+
+      begin
+        track_and_log("#{problem_summary}: #{resolution_summary}") do
+          handler.apply_resolution(resolution)
+        end
+      rescue Bosh::Director::ProblemHandlerError => e
+        log_resolution_error(problem, e)
+      end
+
+      problem.state = "resolved"
+      problem.save
+      @resolved_count += 1
+
+    rescue => e
+      log_resolution_error(problem, e)
+    end
+
+    private
+
+    def log_resolution_error(problem, error)
+      logger.error("Error resolving problem `#{problem.id}': #{error}")
+      logger.error(error.backtrace.join("\n"))
+    end
+  end
+end

data/lib/bosh/director/problem_scanner.rb

@@ -0,0 +1,268 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  class ProblemScanner
+
+    AGENT_TIMEOUT = 10 # seconds
+
+    attr_reader :event_log, :logger
+
+    @queue = :normal
+
+    # @param [String] deployment_name Deployment name
+    def initialize(deployment)
+      @deployment = deployment
+      @instance_manager = Api::InstanceManager.new
+
+      @problem_lock = Mutex.new
+      @agent_disks = {}
+
+      #temp
+      @event_log = Config.event_log
+      @logger = Config.logger
+    end
+
+    def begin_stage(stage_name, n_steps)
+      event_log.begin_stage(stage_name, n_steps)
+      logger.info(stage_name)
+    end
+
+    def track_and_log(task, log = true)
+      event_log.track(task) do |ticker|
+        logger.info(task) if log
+        yield ticker if block_given?
+      end
+    end
+
+    def reset(vms=nil)
+      if vms
+        vms.each do |job, index|
+          instance = @instance_manager.find_by_name(@deployment.name, job, index)
+          Models::DeploymentProblem.where(deployment: deployment,
+                                          :resource_id => instance.vm.id,
+                                          :state => "open").update(state: "closed")
+        end
+      else
+        Models::DeploymentProblem.where(state: "open", deployment: deployment).update(state: "closed")
+      end
+    end
+
+    def scan_disks
+      disks = Models::PersistentDisk.eager(:instance).all.select do |disk|
+        disk.instance && disk.instance.deployment_id == deployment.id
+      end
+      results = Hash.new(0)
+
+      begin_stage("Scanning #{disks.size} persistent disks", 2)
+
+      track_and_log("Looking for inactive disks") do
+        disks.each do |disk|
+          scan_result = scan_disk(disk)
+          results[scan_result] += 1
+        end
+      end
+
+      track_and_log("#{results[:ok]} OK, " +
+                        "#{results[:inactive]} inactive, " +
+                        "#{results[:mount_info_mismatch]} mount-info mismatch")
+    end
+
+    def scan_vms(vms=nil)
+      if vms
+        vm_list = []
+        vms.each do |job, index|
+          instance = @instance_manager.find_by_name(@deployment.name, job, index)
+          vm_list << instance.vm
+        end
+        vms = vm_list
+      else
+        vms = Models::Vm.eager(:instance).filter(deployment: deployment).all
+      end
+
+      begin_stage("Scanning #{vms.size} VMs", 2)
+      results = Hash.new(0)
+      lock = Mutex.new
+
+      track_and_log("Checking VM states") do
+        ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
+          vms.each do |vm|
+            pool.process do
+              scan_result = scan_vm(vm)
+              lock.synchronize { results[scan_result] += 1 }
+            end
+          end
+        end
+      end
+
+      track_and_log("#{results[:ok]} OK, " +
+                        "#{results[:unresponsive]} unresponsive, " +
+                        "#{results[:missing]} missing, " +
+                        "#{results[:unbound]} unbound, " +
+                        "#{results[:out_of_sync]} out of sync")
+    end
+
+    def scan_disk(disk)
+      # inactive disks
+      unless disk.active
+        logger.info("Found inactive disk: #{disk.id}")
+        problem_found(:inactive_disk, disk)
+        return :inactive
+      end
+
+      disk_cid = disk.disk_cid
+      vm_cid = nil
+
+      if disk.instance && disk.instance.vm
+        vm_cid = disk.instance.vm.cid
+      end
+
+      if vm_cid.nil?
+        # With the db dependencies this should not happen.
+        logger.warn("Disk #{disk_cid} is not associated to any VM. " +
+                        "Skipping scan")
+        return :ok
+      end
+
+      owner_vms = get_disk_owners(disk_cid) || []
+      # active disk is not mounted or mounted more than once -or-
+      # the disk is mounted on a vm that is different form the record.
+      if owner_vms.size != 1 || owner_vms.first != vm_cid
+        logger.info("Found problem in mount info: " +
+                        "active disk #{disk_cid} mounted on " +
+                        "#{owner_vms.join(', ')}")
+        problem_found(:mount_info_mismatch, disk, :owner_vms => owner_vms)
+        return :mount_info_mismatch
+      end
+      :ok
+    end
+
+    def scan_vm(vm)
+      agent_options = {
+          :timeout => AGENT_TIMEOUT,
+          :retry_methods => {:get_state => 0}
+      }
+
+      instance = nil
+      mounted_disk_cid = nil
+      @problem_lock.synchronize do
+        instance = vm.instance
+        mounted_disk_cid = instance.persistent_disk_cid if instance
+      end
+
+      agent = AgentClient.new(vm.agent_id, agent_options)
+      begin
+        state = agent.get_state
+
+        # gather mounted disk info. (used by scan_disk)
+        begin
+          disk_list = agent.list_disk
+          mounted_disk_cid = disk_list.first
+        rescue Bosh::Director::RpcTimeout => e
+          mounted_disk_cid = nil
+        rescue RuntimeError => e
+          # For old agents that doesn't implement list_disk we assume the disk is mounted
+          logger.info("agent.list_disk failed on agent #{vm.agent_id}")
+        end
+        add_disk_owner(mounted_disk_cid, vm.cid) if mounted_disk_cid
+
+        return :out_of_sync if is_out_of_sync_vm?(vm, instance, state)
+        return :unbound if is_unbound_instance_vm?(vm, instance, state)
+        :ok
+      rescue Bosh::Director::RpcTimeout
+        # We add the disk to avoid a duplicate problem when timeouts fetching agent status (unresponsive_agent and
+        # mount_info_mismatch)
+        add_disk_owner(mounted_disk_cid, vm.cid) if mounted_disk_cid
+
+        begin
+          unless cloud.has_vm?(vm.cid)
+            logger.info("Missing VM #{vm.cid}")
+            problem_found(:missing_vm, vm)
+            return :missing
+          end
+        rescue Bosh::Clouds::NotImplemented
+        end
+
+        logger.info("Found unresponsive agent #{vm.agent_id}")
+        problem_found(:unresponsive_agent, vm)
+        :unresponsive
+      end
+    end
+
+    def problem_found(type, resource, data = {})
+      @problem_lock.synchronize do
+        similar_open_problems = Models::DeploymentProblem.
+            filter(:deployment_id => deployment.id, :type => type.to_s,
+                   :resource_id => resource.id, :state => "open").all
+
+        if similar_open_problems.size > 1
+          raise CloudcheckTooManySimilarProblems,
+                "More than one problem of type `#{type}' " +
+                    "exists for resource #{type} #{resource.id}"
+        end
+
+        if similar_open_problems.empty?
+          problem = Models::DeploymentProblem.
+              create(:type => type.to_s, :resource_id => resource.id,
+                     :state => "open", :deployment_id => deployment.id,
+                     :data => data, :counter => 1)
+
+          logger.info("Created problem #{problem.id} (#{problem.type})")
+        else
+          # This assumes we are running with deployment lock acquired,
+          # so there is no possible update conflict
+          problem = similar_open_problems[0]
+          problem.data = data
+          problem.last_seen_at = Time.now
+          problem.counter += 1
+          problem.save
+          logger.info("Updated problem #{problem.id} (#{problem.type}), " +
+                          "count is now #{problem.counter}")
+        end
+      end
+    end
+
+
+    private
+    attr_reader :deployment
+
+    def is_out_of_sync_vm?(vm, instance, state)
+      job = state["job"] ? state["job"]["name"] : nil
+      index = state["index"]
+      if state["deployment"] != deployment.name ||
+          (instance && (instance.job != job || instance.index != index))
+        problem_found(:out_of_sync_vm, vm,
+                      :deployment => state["deployment"],
+                      :job => job, :index => index)
+        true
+      else
+        false
+      end
+    end
+
+    def is_unbound_instance_vm?(vm, instance, state)
+      job = state["job"] ? state["job"]["name"] : nil
+      index = state["index"]
+      if job && !instance
+        logger.info("Found unbound VM #{vm.agent_id}")
+        problem_found(:unbound_instance_vm, vm,
+                      :job => job, :index => index)
+        true
+      else
+        false
+      end
+    end
+
+    def add_disk_owner(disk_cid, vm_cid)
+      @agent_disks[disk_cid] ||= []
+      @agent_disks[disk_cid] << vm_cid
+    end
+
+    def get_disk_owners(disk_cid)
+      @agent_disks[disk_cid]
+    end
+
+    def cloud
+      Config.cloud
+    end
+  end
+end