bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/instance_reuser.rb

@@ -0,0 +1,89 @@
+module Bosh::Director
+  class NilInstanceError < ArgumentError; end
+
+  # A class for maintaining Instance objects, making reusing Instances easier.
+  class InstanceReuser
+    def initialize
+      @idle_instances_by_stemcell = {}
+      @in_use_instances_by_stemcell = {}
+      @mutex = Mutex.new
+    end
+
+    # Adds an instance's information to the pool of VMs that can be reused.
+    # @param [DeploymentPlan::Instance] The instance for the new VM.
+    def add_in_use_instance(instance, stemcell)
+      raise NilInstanceError if instance.nil?
+      @mutex.synchronize do
+        @in_use_instances_by_stemcell[stemcell] ||= []
+        @in_use_instances_by_stemcell[stemcell] << instance
+      end
+    end
+
+    # Returns the instance of a VM that is not in use and can be reused.
+    # @param [Models::Stemcell] stemcell The stemcell that the VM must be running.
+    # @return [DeploymentPlan::Instance] The instance for an existing unused VM, if one exists. Otherwise, nil.
+    def get_instance(stemcell)
+      @mutex.synchronize do
+        return nil if @idle_instances_by_stemcell[stemcell].nil?
+        instance = @idle_instances_by_stemcell[stemcell].pop
+        return nil if instance.nil?
+        @in_use_instances_by_stemcell[stemcell] ||= []
+        @in_use_instances_by_stemcell[stemcell] << instance
+        return instance
+      end
+    end
+
+    def release_instance(instance)
+      raise NilInstanceError if instance.nil?
+      @mutex.synchronize do
+        release_without_lock(instance)
+      end
+    end
+
+    def remove_instance(instance)
+      raise NilInstanceError if instance.nil?
+      @mutex.synchronize do
+        release_without_lock(instance)
+        @idle_instances_by_stemcell.each_value do |vms|
+          vms.each do |v|
+            vms.delete(v) if instance == v
+          end
+        end
+      end
+    end
+
+    # Gets the total number of compilation instances created with a given stemcell.
+    # @param [Models::Stemcell] stemcell The stemcell the VMs are running.
+    # @return [Integer] The number of instances running a given stemcell.
+    def get_num_instances(stemcell)
+      @mutex.synchronize do
+        idle_count = @idle_instances_by_stemcell[stemcell].nil? ? 0 : @idle_instances_by_stemcell[stemcell].size
+        in_use_count = @in_use_instances_by_stemcell[stemcell].nil? ? 0 : @in_use_instances_by_stemcell[stemcell].size
+        idle_count + in_use_count
+      end
+    end
+
+    # An iterator for all compilation VMs on all stemcells.
+    # @yield [DeploymentPlan::Instance] yields each instance in InstanceReuser.
+    def each
+      all_vms = (@idle_instances_by_stemcell.values + @in_use_instances_by_stemcell.values).flatten
+      all_vms.each do |vm|
+        yield vm
+      end
+    end
+
+    private
+
+    def release_without_lock(instance)
+      @in_use_instances_by_stemcell.each do |stemcell, vms|
+        vms.each do |v|
+          if instance == v
+            vms.delete(v)
+            @idle_instances_by_stemcell[stemcell] ||= []
+            @idle_instances_by_stemcell[stemcell] << instance
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/instance_updater.rb

@@ -2,283 +2,142 @@ require 'bosh/director/rendered_job_templates_cleaner'
 
 module Bosh::Director
   class InstanceUpdater
-    include DnsHelper
-
     WATCH_INTERVALS = 10
+    MAX_RECREATE_ATTEMPTS = 3
 
     attr_reader :current_state
 
-    # @params [DeploymentPlan::Instance] instance
-    def initialize(instance, event_log_task, job_renderer)
-      @instance = instance
-      @event_log_task = event_log_task
-      @job_renderer = job_renderer
-
-      @cloud = Config.cloud
-      @logger = Config.logger
-      @blobstore = App.instance.blobstores.blobstore
-
-      @job = instance.job
-      @target_state = @instance.state
-
-      @deployment_plan = @job.deployment
-      @resource_pool = @job.resource_pool
-      @update_config = @job.update
-
-      @vm = @instance.model.vm
-
+    def self.new_instance_updater(ip_provider)
+      logger = Config.logger
+      cloud = Config.cloud
+      vm_deleter = VmDeleter.new(cloud, logger)
+      disk_manager = DiskManager.new(cloud, logger)
+      job_renderer = JobRenderer.create
+      vm_creator = VmCreator.new(cloud, logger, vm_deleter, disk_manager, job_renderer)
+      vm_recreator = VmRecreator.new(vm_creator, vm_deleter)
+      dns_manager = DnsManager.create
+      new(
+        cloud,
+        logger,
+        ip_provider,
+        App.instance.blobstores.blobstore,
+        vm_deleter,
+        vm_creator,
+        dns_manager,
+        disk_manager,
+        vm_recreator
+      )
+    end
+
+    def initialize(cloud, logger, ip_provider, blobstore, vm_deleter, vm_creator, dns_manager, disk_manager, vm_recreator)
+      @cloud = cloud
+      @logger = logger
+      @blobstore = blobstore
+      @vm_deleter = vm_deleter
+      @vm_creator = vm_creator
+      @dns_manager = dns_manager
+      @disk_manager = disk_manager
+      @ip_provider = ip_provider
       @current_state = {}
-
-      @agent = AgentClient.with_defaults(@vm.agent_id)
+      @vm_recreator = vm_recreator
     end
 
-    def report_progress(num_steps)
-      @event_log_task.advance(100.0 / num_steps)
-    end
+    def update(instance_plan, options = {})
+      instance = instance_plan.instance
+      @logger.info("Updating instance #{instance}, changes: #{instance_plan.changes.to_a.join(', ').inspect}")
 
-    def update_steps(options = {})
-      steps = []
       @canary = options.fetch(:canary, false)
 
       # Optimization to only update DNS if nothing else changed.
-      if dns_change_only?
-        steps << proc { update_dns }
-        return steps
+      if dns_change_only?(instance_plan)
+        @logger.debug('Only change is DNS configuration')
+        update_dns(instance_plan)
+        return
       end
 
-      steps << proc { Preparer.new(@instance, agent, @logger).prepare }
-      steps << proc { stop }
-      steps << proc { take_snapshot }
-
-      if @target_state == "detached"
-        steps << proc { vm_updater.detach }
-        return steps
-      end
+      unless instance_plan.currently_detached?
+        Preparer.new(instance_plan, agent(instance), @logger).prepare
 
-      steps << proc { recreate_vm(nil) }
-      steps << proc { update_networks }
-      steps << proc { update_dns }
-      steps << proc { update_persistent_disk }
-      steps << proc { update_settings }
-
-      if !trusted_certs_change_only?
-        steps << proc {
-          VmMetadataUpdater.build.update(@vm, {})
-          apply_state(@instance.spec)
-          RenderedJobTemplatesCleaner.new(@instance.model, @blobstore, @logger).clean
-        }
+        stop(instance_plan)
+        take_snapshot(instance)
       end
 
-      if need_start?
-        steps << proc { run_pre_start_scripts }
-        steps << proc { start! }
+      if instance.state == 'detached'
+        @logger.info("Detaching instance #{instance}")
+        unless instance_plan.currently_detached?
+          @disk_manager.unmount_disk_for(instance_plan)
+          @vm_deleter.delete_for_instance_plan(instance_plan)
+        end
+        release_obsolete_ips(instance_plan)
+        instance.update_state
+        return
       end
 
-      steps << proc { wait_until_running }
-
-      steps
-    end
-
-    def update(options = {})
-      steps = update_steps(options)
-
-      @logger.info("Updating instance #{@instance}, changes: #{@instance.changes.to_a.join(', ')}")
-
-      steps.each do |step|
-        step.call
-        report_progress(steps.length)
+      recreated = false
+      if needs_recreate?(instance_plan)
+        @logger.debug('Failed to update in place. Recreating VM')
+        @disk_manager.unmount_disk_for(instance_plan)
+        @vm_recreator.recreate_vm(instance_plan, nil)
+        recreated = true
       end
 
-      if @target_state == "started" && current_state["job_state"] != "running"
-        raise AgentJobNotRunning, "`#{@instance}' is not running after update"
-      end
+      release_obsolete_ips(instance_plan)
 
-      if @target_state == "stopped" && current_state["job_state"] == "running"
-        raise AgentJobNotStopped, "`#{@instance}' is still running despite the stop command"
-      end
-    end
+      update_dns(instance_plan)
+      @disk_manager.update_persistent_disk(instance_plan, @vm_recreator)
 
-    # Watch times don't include the get_state roundtrip time, so effective
-    # max watch time is roughly:
-    # max_watch_time + N_WATCH_INTERVALS * avg_roundtrip_time
-    def wait_until_running
-      watch_schedule(min_watch_time, max_watch_time).each do |watch_time|
-        sleep_time = watch_time.to_f / 1000
-        @logger.info("Waiting for #{sleep_time} seconds to check #{@instance} status")
-        sleep(sleep_time)
-        @logger.info("Checking if #{@instance} has been updated after #{sleep_time} seconds")
-
-        @current_state = agent.get_state
-
-        if @target_state == "started"
-          break if current_state["job_state"] == "running"
-        elsif @target_state == "stopped"
-          break if current_state["job_state"] != "running"
+      unless recreated
+        if instance.trusted_certs_changed?
+          @logger.debug('Updating trusted certs')
+          instance.update_trusted_certs
         end
       end
-    end
-
-    def run_pre_start_scripts
-      @agent.run_script("pre-start", {})
-    end
-
-    def start!
-      agent.start
-    end
-
-    def need_start?
-      @target_state == 'started'
-    end
-
-    def dns_change_only?
-      @instance.changes.include?(:dns) && @instance.changes.size == 1
-    end
-
-    def trusted_certs_change_only?
-      @instance.changes.include?(:trusted_certs) && @instance.changes.size == 1
-    end
-
-    def stop
-      skip_drain = @deployment_plan.skip_drain_for_job?(@job.name)
-      stopper = Stopper.new(@instance, agent, @target_state, skip_drain, Config, @logger)
-      stopper.stop
-    end
-
-    def take_snapshot
-      Api::SnapshotManager.take_snapshot(@instance.model, clean: true)
-    end
-
-    def delete_snapshots(disk)
-      Api::SnapshotManager.delete_snapshots(disk.snapshots)
-    end
-
-    def apply_state(state)
-      @vm.update(:apply_spec => state)
-      agent.apply(state)
-    end
-
-    # Retrieve list of mounted disks from the agent
-    # @return [Array<String>] list of disk CIDs
-    def disk_info
-      return @disk_list if @disk_list
-
-      begin
-        @disk_list = agent.list_disk
-      rescue RuntimeError
-        # old agents don't support list_disk rpc
-        [@instance.persistent_disk_cid]
-      end
-    end
 
-    def delete_unused_disk(disk)
-      @cloud.delete_disk(disk.disk_cid)
-      disk.destroy
-    end
+      cleaner = RenderedJobTemplatesCleaner.new(instance.model, @blobstore, @logger)
+      InstanceUpdater::StateApplier.new(instance_plan, agent(instance), cleaner).apply
 
-    def delete_mounted_disk(disk)
-      disk_cid = disk.disk_cid
-      vm_cid = @vm.cid
+      instance.update_state
 
-      # Unmount the disk only if disk is known by the agent
-      if agent && disk_info.include?(disk_cid)
-        agent.unmount_disk(disk_cid)
-      end
+      wait_until_running(instance_plan)
 
-      begin
-        @cloud.detach_disk(vm_cid, disk_cid) if vm_cid
-      rescue Bosh::Clouds::DiskNotAttached
-        if disk.active
-          raise CloudDiskNotAttached,
-                "`#{@instance}' VM should have persistent disk attached " +
-                    "but it doesn't (according to CPI)"
-        end
+      if instance.state == "started" && current_state["job_state"] != "running"
+        raise AgentJobNotRunning, "`#{instance}' is not running after update"
       end
 
-      delete_snapshots(disk)
-
-      begin
-        @cloud.delete_disk(disk_cid)
-      rescue Bosh::Clouds::DiskNotFound
-        if disk.active
-          raise CloudDiskMissing,
-                "Disk `#{disk_cid}' is missing according to CPI but marked " +
-                    "as active in DB"
-        end
+      if instance.state == "stopped" && current_state["job_state"] == "running"
+        raise AgentJobNotStopped, "`#{instance}' is still running despite the stop command"
       end
-
-      disk.destroy
     end
 
-    def update_dns
-      return unless @instance.dns_changed?
+    private
 
-      domain = @deployment_plan.dns_domain
-      @instance.dns_record_info.each do |record_name, ip_address|
-        @logger.info("Updating DNS for: #{record_name} to #{ip_address}")
-        update_dns_a_record(domain, record_name, ip_address)
-        update_dns_ptr_record(record_name, ip_address)
+    def release_obsolete_ips(instance_plan)
+      instance_plan.network_plans
+        .select(&:obsolete?)
+        .each do |network_plan|
+        reservation = network_plan.reservation
+        @ip_provider.release(reservation)
       end
-      flush_dns_cache
+      instance_plan.release_obsolete_network_plans
     end
 
-    def recreate_vm(new_disk_cid)
-      @vm, @agent = vm_updater.update(new_disk_cid)
+    def stop(instance_plan)
+      instance = instance_plan.instance
+      stopper = Stopper.new(instance_plan, instance.state, Config, @logger)
+      stopper.stop
     end
 
-    # Synchronizes persistent_disks with the agent.
-    # (Currently assumes that we only have 1 persistent disk.)
-    # @return [void]
-    def check_persistent_disk
-      return if @instance.model.persistent_disks.empty?
-      agent_disk_cid = disk_info.first
-
-      if agent_disk_cid != @instance.model.persistent_disk_cid
-        raise AgentDiskOutOfSync,
-              "`#{@instance}' has invalid disks: agent reports " +
-                  "`#{agent_disk_cid}' while director record shows " +
-                  "`#{@instance.model.persistent_disk_cid}'"
-      end
-
-      @instance.model.persistent_disks.each do |disk|
-        unless disk.active
-          @logger.warn("`#{@instance}' has inactive disk #{disk.disk_cid}")
-        end
-      end
+    def take_snapshot(instance)
+      Api::SnapshotManager.take_snapshot(instance.model, clean: true)
     end
 
-    def update_persistent_disk
-      vm_updater.attach_missing_disk
-      check_persistent_disk
+    def update_dns(instance_plan)
+      instance = instance_plan.instance
 
-      disk = nil
-      return unless @instance.persistent_disk_changed?
+      return unless instance_plan.dns_changed?
 
-      old_disk = @instance.model.persistent_disk
-
-      if @job.persistent_disk_pool && @job.persistent_disk_pool.disk_size > 0
-        disk = create_disk
-        attach_disk(disk)
-        mount_and_migrate_disk(disk, old_disk)
-      end
-
-      @instance.model.db.transaction do
-        old_disk.update(:active => false) if old_disk
-        disk.update(:active => true) if disk
-      end
-
-      delete_mounted_disk(old_disk) if old_disk
-    end
-
-    def update_networks
-      network_updater = NetworkUpdater.new(@instance, @vm, agent, vm_updater, @cloud, @logger)
-      @vm, @agent = network_updater.update
-    end
-
-    def update_settings
-      if @instance.trusted_certs_changed?
-        @agent.update_settings(Config.trusted_certs)
-        @vm.update(:trusted_certs_sha1 => Digest::SHA1.hexdigest(Config.trusted_certs))
-      end
+      @dns_manager.update_dns_record_for_instance(instance.model, instance_plan.network_settings.dns_record_info)
+      @dns_manager.flush_dns_cache
     end
 
     # Returns an array of wait times distributed
@@ -299,70 +158,69 @@ module Bosh::Director
       [min_watch_time] + ([step] * (delta / step).floor)
     end
 
-    def min_watch_time
-      canary? ? @update_config.min_canary_watch_time : @update_config.min_update_watch_time
+    def get_min_watch_time(update_config)
+      canary? ? update_config.min_canary_watch_time : update_config.min_update_watch_time
     end
 
-    def max_watch_time
-      canary? ? @update_config.max_canary_watch_time : @update_config.max_update_watch_time
+    def get_max_watch_time(update_config)
+      canary? ? update_config.max_canary_watch_time : update_config.max_update_watch_time
     end
 
     def canary?
       @canary
     end
 
-    attr_reader :agent
-
-    def vm_updater
-      # Do not memoize to avoid caching same VM and agent
-      # which could be replaced after updating a VM
-      VmUpdater.new(@instance, @vm, agent, @job_renderer, @cloud, 3, @logger)
+    def dns_change_only?(instance_plan)
+      instance_plan.changes.include?(:dns) && instance_plan.changes.size == 1
     end
 
-    private
+    # Watch times don't include the get_state roundtrip time, so effective
+    # max watch time is roughly:
+    # max_watch_time + N_WATCH_INTERVALS * avg_roundtrip_time
+    def wait_until_running(instance_plan)
+      instance = instance_plan.instance
+      job = instance_plan.desired_instance.job
+      min_watch_time = get_min_watch_time(job.update)
+      max_watch_time = get_max_watch_time(job.update)
+      watch_schedule(min_watch_time, max_watch_time).each do |watch_time|
+        sleep_time = watch_time.to_f / 1000
+        @logger.info("Waiting for #{sleep_time} seconds to check #{instance} status")
+        sleep(sleep_time)
+        @logger.info("Checking if #{instance} has been updated after #{sleep_time} seconds")
 
-    def create_disk
-      disk_size = @job.persistent_disk_pool.disk_size
-      cloud_properties = @job.persistent_disk_pool.cloud_properties
-
-      disk = nil
-      @instance.model.db.transaction do
-        disk_cid = @cloud.create_disk(disk_size, cloud_properties, @vm.cid)
-        disk = Models::PersistentDisk.create(
-          disk_cid: disk_cid,
-          active: false,
-          instance_id: @instance.model.id,
-          size: disk_size,
-          cloud_properties: cloud_properties,
-        )
+        @current_state = agent(instance).get_state
+
+        if instance.state == "started"
+          break if current_state["job_state"] == "running"
+        elsif instance.state == "stopped"
+          break if current_state["job_state"] != "running"
+        end
       end
-      disk
     end
 
-    def attach_disk(disk)
-      @cloud.attach_disk(@vm.cid, disk.disk_cid)
-    rescue Bosh::Clouds::NoDiskSpace => e
-      if e.ok_to_retry
-        @logger.warn('Retrying attach disk operation after persistent disk update failed')
-        recreate_vm(disk.disk_cid)
-        begin
-          @cloud.attach_disk(@vm.cid, disk.disk_cid)
-        rescue
-          delete_unused_disk(disk)
-          raise
-        end
-      else
-        delete_unused_disk(disk)
-        raise
+    def needs_recreate?(instance_plan)
+      instance = instance_plan.instance
+
+      if instance_plan.needs_shutting_down?
+        @logger.debug('VM needs to be shutdown before it can be updated.')
+        return true
       end
+
+      if instance.cloud_properties_changed?
+        @logger.debug('Cloud Properties have changed. Recreating VM')
+        return true
+      end
+
+      if instance_plan.networks_changed?
+        @logger.debug('Networks have changed. Recreating VM')
+        return true
+      end
+
+      false
     end
 
-    def mount_and_migrate_disk(new_disk, old_disk)
-      agent.mount_disk(new_disk.disk_cid)
-      agent.migrate_disk(old_disk.disk_cid, new_disk.disk_cid) if old_disk
-    rescue
-      delete_mounted_disk(new_disk)
-      raise
+    def agent(instance)
+      AgentClient.with_vm(instance.model.vm)
     end
   end
 end