bosh-director 1.5.0.pre.1113

data/lib/bosh/director/problem_handlers/base.rb

@@ -0,0 +1,153 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ProblemHandlers
+
+    class Base
+      include CloudcheckHelper
+
+      attr_reader :data
+      attr_accessor :job # so we can checkpoint task
+
+      def self.create_from_model(model)
+        create_by_type(model.type, model.resource_id, model.data)
+      end
+
+      # create_by_type might not be able to initialize
+      # problem handler if some part of its state
+      # or dependencies is invalid. In this case it
+      # just substitutes a generic "invalid_problem"
+      # handler that reports the fact that the original
+      # problem is invalid and offers closing it as
+      # the only solution.
+      def self.create_by_type(type, resource_id, data)
+        handler_class = Base.handlers[type.to_s]
+        if handler_class.nil?
+          raise "Cannot find handler for `#{type}' problem"
+        end
+
+        handler_class.new(resource_id, data)
+      rescue ProblemHandlerError => e
+        create_by_type(:invalid_problem, resource_id,
+                       {"error" => e, "original_type" => type.to_s})
+      end
+
+      # Problem state is  described by constructor parameters.
+      # Problem handler can reach out to check if the problem
+      # is still present and attempt to fix it by applying
+      # a potential resolution tagged with one or more labels.
+      def initialize(resource_id, data)
+        @logger = Config.logger
+        @event_log = Config.event_log
+        @job = nil
+      end
+
+      def checkpoint
+        @job.task_checkpoint if @job
+      end
+
+      # Talking to cloud should only be possible
+      # in the context of Resque job
+      def cloud
+        if @job.nil?
+          handler_error("Cannot talk to cloud outside of job context")
+        end
+        super
+      end
+
+      # Problem description
+      def description; end
+
+      def resolutions
+        self.class.resolutions.map do |name|
+          { :name => name.to_s, :plan => resolution_plan(name) }
+        end
+      end
+
+      def resolution_plan(resolution)
+        plan = self.class.plan_for(resolution)
+        return nil if plan.nil?
+        instance_eval(&plan)
+      end
+
+      def auto_resolution
+        self.class.get_auto_resolution
+      end
+
+      # @param resolution desired resolution
+      def apply_resolution(resolution)
+        action = self.class.action_for(resolution)
+        if action.nil?
+          handler_error("Cannot find `#{resolution}' resolution for `#{self.class}'")
+        end
+        instance_eval(&action)
+      end
+
+      def auto_resolve
+        apply_resolution(auto_resolution)
+      end
+
+      # Registration DSL
+      class << self
+        attr_accessor :handlers
+      end
+
+      def self.register_as(type)
+        Base.handlers ||= {}
+        Base.handlers[type.to_s] = self
+      end
+
+      # Resolution DSL
+      class << self
+        attr_reader :resolutions
+      end
+
+      def self.init_dsl_data
+        @resolutions = []
+        @plans = {}
+        @actions = {}
+        @auto_resolution = nil
+      end
+
+      init_dsl_data
+
+      def self.inherited(base)
+        base.class_eval { init_dsl_data }
+      end
+
+      def self.plan_for(resolution)
+        @plans[resolution.to_s]
+      end
+
+      def self.action_for(resolution)
+        @actions[resolution.to_s]
+      end
+
+      def self.plan(&block)
+        @plans[@pending_name.to_s] = block
+      end
+
+      def self.action(&block)
+        @actions[@pending_name.to_s] = block
+      end
+
+      def self.get_auto_resolution
+        @auto_resolution
+      end
+
+      def self.auto_resolution(name)
+        @auto_resolution = name
+      end
+
+      def self.resolution(name, &block)
+        @resolutions << name
+        @pending_name = name
+        instance_eval(&block)
+      ensure
+        @pending_name = nil
+      end
+
+    end
+  end
+end
+

data/lib/bosh/director/problem_handlers/inactive_disk.rb

@@ -0,0 +1,112 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ProblemHandlers
+    class InactiveDisk < Base
+
+      register_as :inactive_disk
+      auto_resolution :ignore
+
+      def initialize(disk_id, data)
+        super
+        @disk_id = disk_id
+        @data = data
+        @disk = Models::PersistentDisk[@disk_id]
+
+        if @disk.nil?
+          handler_error("Disk `#{@disk_id}' is no longer in the database")
+        end
+
+        if @disk.active
+          handler_error("Disk `#{@disk.disk_cid}' is no longer inactive")
+        end
+
+        @instance = @disk.instance
+        if @instance.nil?
+          handler_error("Cannot find instance for disk `#{@disk.disk_cid}'")
+        end
+
+        @vm = @instance.vm
+      end
+
+      def description
+        job = @instance.job || "unknown job"
+        index = @instance.index || "unknown index"
+        disk_label = "`#{@disk.disk_cid}' (#{job}/#{index}, #{@disk.size.to_i}M)"
+        "Disk #{disk_label} is inactive"
+      end
+
+      resolution :ignore do
+        plan { "Ignore problem" }
+        action { }
+      end
+
+      resolution :delete_disk do
+        plan { "Delete disk" }
+        action { delete_disk }
+      end
+
+      resolution :activate_disk do
+        plan { "Activate disk" }
+        action { activate_disk }
+      end
+
+      def activate_disk
+        unless disk_mounted?
+          handler_error("Disk is not mounted")
+        end
+        # Currently the director allows ONLY one persistent disk per
+        # instance. We are about to activate a disk but the instance already
+        # has an active disk.
+        # For now let's be conservative and return an error.
+        if @instance.persistent_disk
+          handler_error("Instance already has an active disk")
+        end
+        @disk.active = true
+        @disk.save
+      end
+
+      def delete_disk
+        if disk_mounted?
+          handler_error("Disk is currently in use")
+        end
+
+        if @vm
+          begin
+            cloud.detach_disk(@vm.cid, @disk.disk_cid)
+          rescue => e
+            # We are going to delete this disk anyway
+            # and we know it's not in use, so we can ignore
+            # detach errors here.
+            @logger.warn(e)
+          end
+        end
+
+        # FIXME: Currently there is no good way to know if delete_disk
+        # failed because of cloud error or because disk doesn't exist
+        # in vsphere_disks.
+        begin
+          cloud.delete_disk(@disk.disk_cid)
+        rescue Bosh::Clouds::DiskNotFound, RuntimeError => e # FIXME
+          @logger.warn(e)
+        end
+
+        @disk.destroy
+      end
+
+      def disk_mounted?
+        return false if @vm.nil?
+
+        begin
+          agent_timeout_guard(@vm) do |agent|
+            agent.list_disk.include?(@disk.disk_cid)
+          end
+        rescue RuntimeError
+          # old stemcells without 'list_disk' support. We need to play
+          # conservative and assume that the disk is mounted.
+          true
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/problem_handlers/invalid_problem.rb

@@ -0,0 +1,28 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ProblemHandlers
+    class InvalidProblem < Base
+
+      register_as :invalid_problem
+      auto_resolution :close
+
+      def initialize(resource_id, data)
+        super
+        @resource_id = resource_id
+        @error = data["error"] || "unknown error"
+        @original_type = data["original_type"] || "unknown_type"
+      end
+
+      def description
+        "Problem (#{@original_type} #{@resource_id}) is no longer valid: #{@error}"
+      end
+
+      resolution :close do
+        plan { "Close problem" }
+        action { }
+      end
+
+    end
+  end
+end

data/lib/bosh/director/problem_handlers/missing_vm.rb

@@ -0,0 +1,34 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ProblemHandlers
+    class MissingVM < Base
+
+      register_as :missing_vm
+
+      def initialize(vm_id, data)
+        super
+        @vm = Models::Vm[vm_id]
+      end
+
+      resolution :ignore do
+        plan { "Ignore problem" }
+        action { }
+      end
+
+      resolution :recreate_vm do
+        plan { "Recreate VM using last known apply spec" }
+        action { recreate_vm(@vm) }
+      end
+
+      resolution :delete_vm_reference do
+        plan { "Delete VM reference (DANGEROUS!)" }
+        action { delete_vm_reference(@vm, skip_cid_check: true) }
+      end
+
+      def description
+        "VM with cloud ID `#{@vm.cid}' missing."
+      end
+    end
+  end
+end

data/lib/bosh/director/problem_handlers/mount_info_mismatch.rb

@@ -0,0 +1,62 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ProblemHandlers
+    class MountInfoMismatch < Base
+
+      register_as :mount_info_mismatch
+      auto_resolution :ignore
+
+      def initialize(disk_id, data)
+        @disk = Models::PersistentDisk[disk_id]
+        @data = data
+
+        if @disk.nil?
+          handler_error("Disk `#{disk_id}' is no longer in the database")
+        end
+
+        @disk_cid = @disk.disk_cid
+        @vm_cid = @disk.instance.vm.cid if @disk.instance && @disk.instance.vm
+        handler_error("Can't find corresponding vm-cid for disk `#{@disk_cid}'") if @vm_cid.nil?
+
+        @instance = @disk.instance
+        @vm = @instance.vm
+
+        @disk_owners = @data['owner_vms']
+      end
+
+      def description
+        out =  "Inconsistent mount information:\n"
+        out += "Record shows that disk '#{@disk_cid}' should be mounted on #{@vm_cid}.\n"
+        out += "However it is currently :\n"
+
+        if @disk_owners.size == 0
+          out += "\tNot mounted in any VM"
+        else
+          out += "\tMounted on: #{@disk_owners.join(", ")}"
+        end
+        out
+      end
+
+      resolution :ignore do
+        plan { "Ignore" }
+        action { }
+      end
+
+      resolution :reattach_disk do
+        plan { "Reattach disk to instance" }
+        action { reattach_disk(false) }
+      end
+
+      resolution :reattach_disk_and_reboot do
+        plan { "Reattach disk and reboot instance" }
+        action { reattach_disk(true) }
+      end
+      
+      def reattach_disk(reboot = false)
+        cloud.attach_disk(@vm_cid, @disk_cid)
+        reboot ? reboot_vm(@vm) : agent_timeout_guard(@vm) { |agent| agent.mount_disk(@disk_cid) }
+      end
+    end
+  end
+end

data/lib/bosh/director/problem_handlers/out_of_sync_vm.rb

@@ -0,0 +1,64 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module Bosh::Director
+  module ProblemHandlers
+    class OutOfSyncVm < Base
+
+      register_as :out_of_sync_vm
+      auto_resolution :ignore
+
+      def initialize(vm_id, data)
+        super
+        @vm = Models::Vm[vm_id]
+        @data = data
+
+        if @vm.nil?
+          handler_error("VM `#{vm_id}' is no longer in the database")
+        end
+
+        @deployment = @vm.deployment
+        @instance = @vm.instance
+
+        if @deployment.nil?
+          handler_error("VM `#{@vm.cid}' doesn't belong to any deployment")
+        end
+
+      end
+
+      def description
+        actual_deployment = @data["deployment"] || "unknown deployment"
+        actual_job = @data["job"] || "unknown job"
+        actual_index = @data["index"] || "unknown index"
+
+        expected = "#{@deployment.name}: #{instance_name(@vm)}"
+        actual = "#{actual_deployment}: #{actual_job}/#{actual_index}"
+
+        "VM `#{@vm.cid}' is out of sync: expected `#{expected}', got `#{actual}'"
+      end
+
+      resolution :ignore do
+        plan { "Ignore problem" }
+        action { }
+      end
+
+      resolution :delete_vm do
+        plan { "Delete VM (unless it has persistent disk)"}
+        action { validate; delete_vm(@vm) }
+      end
+
+      def validate
+        state = agent_timeout_guard(@vm) { |agent | agent.get_state }
+        return if state["deployment"] != @deployment.name
+
+        # VM is no longer out of sync if no instance is referencing it,
+        # as this situation can actually be handled by regular deployment
+        if @instance.nil? ||
+            state["job"] && state["job"]["name"] == @instance.job &&
+            state["index"] == @instance.index
+          handler_error("VM is now back in sync")
+        end
+      end
+
+    end
+  end
+end