bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/problem_handlers/inactive_disk.rb

@@ -31,8 +31,9 @@ module Bosh::Director
 
       def description
         job = @instance.job || "unknown job"
+        uuid = @instance.uuid || "unknown id"
         index = @instance.index || "unknown index"
-        disk_label = "`#{@disk.disk_cid}' (#{job}/#{index}, #{@disk.size.to_i}M)"
+        disk_label = "`#{@disk.disk_cid}' (#{@disk.size.to_i}M) for instance `#{job}/#{uuid} (#{index})'"
         "Disk #{disk_label} is inactive"
       end
 
@@ -82,29 +83,13 @@ module Bosh::Director
           end
         end
 
-        # FIXME: Currently there is no good way to know if delete_disk
-        # failed because of cloud error or because disk doesn't exist
-        # in vsphere_disks.
-        begin
-          cloud.delete_disk(@disk.disk_cid)
-        rescue Bosh::Clouds::DiskNotFound, RuntimeError => e # FIXME
-          @logger.warn(e)
-        end
-
-        @disk.destroy
+        DiskManager.new(cloud, @logger).orphan_disk(@disk)
       end
 
       def disk_mounted?
         return false if @vm.nil?
-
-        begin
-          agent_timeout_guard(@vm) do |agent|
-            agent.list_disk.include?(@disk.disk_cid)
-          end
-        rescue RuntimeError
-          # old stemcells without 'list_disk' support. We need to play
-          # conservative and assume that the disk is mounted.
-          true
+        agent_timeout_guard(@vm) do |agent|
+          agent.list_disk.include?(@disk.disk_cid)
         end
       end
     end

data/lib/bosh/director/problem_handlers/missing_disk.rb

@@ -41,9 +41,7 @@ module Bosh::Director
       end
 
       def delete_disk_reference
-        @disk.db.transaction do
-          @disk.update(active: false)
-        end
+        @disk.update(active: false)
 
         # If VM is present we try to unmount and detach disk from VM
         if @vm && @vm.cid && cloud.has_vm?(@vm.cid)
@@ -74,17 +72,8 @@ module Bosh::Director
           end
         end
 
-        @logger.debug('Deleting disk snapshots')
-        Api::SnapshotManager.delete_snapshots(@disk.snapshots)
-
-        begin
-          @logger.debug('Sending cpi request: delete_disk')
-          cloud.delete_disk(@disk.disk_cid)
-        rescue Bosh::Clouds::DiskNotFound
-        end
+        DiskManager.new(cloud, @logger).orphan_disk(@disk)
 
-        @logger.debug('Removing disk reference from database')
-        @disk.destroy
       end
     end
   end

data/lib/bosh/director/problem_resolver.rb

@@ -54,7 +54,7 @@ module Bosh::Director
       resolution = @resolutions[problem.id.to_s] || handler.auto_resolution
       problem_summary = "#{problem.type} #{problem.resource_id}"
       resolution_summary = handler.resolution_plan(resolution)
-      resolution_summary ||= "no resolution"
+      resolution_summary ||= 'no resolution'
 
       begin
         track_and_log("#{problem_summary}: #{resolution_summary}") do
@@ -64,7 +64,7 @@ module Bosh::Director
         log_resolution_error(problem, e)
       end
 
-      problem.state = "resolved"
+      problem.state = 'resolved'
       problem.save
       @resolved_count += 1
 

data/lib/bosh/director/problem_scanner/vm_scan_stage.rb

@@ -45,8 +45,7 @@ module Bosh::Director
       @event_logger.track_and_log("#{results[:ok]} OK, " +
         "#{results[:unresponsive]} unresponsive, " +
         "#{results[:missing]} missing, " +
-        "#{results[:unbound]} unbound, " +
-        "#{results[:out_of_sync]} out of sync")
+        "#{results[:unbound]} unbound")
     end
 
     private
@@ -59,7 +58,7 @@ module Bosh::Director
 
       instance, mounted_disk_cid = @problem_register.get_vm_instance_and_disk(vm)
 
-      agent = AgentClient.with_defaults(vm.agent_id, agent_options)
+      agent = AgentClient.with_vm(vm, agent_options)
       begin
         state = agent.get_state
 
@@ -69,13 +68,9 @@ module Bosh::Director
           mounted_disk_cid = disk_list.first
         rescue Bosh::Director::RpcTimeout
           mounted_disk_cid = nil
-        rescue RuntimeError
-          # For old agents that doesn't implement list_disk we assume the disk is mounted
-          @logger.info("agent.list_disk failed on agent #{vm.agent_id}")
         end
         add_disk_owner(mounted_disk_cid, vm.cid) if mounted_disk_cid
 
-        return :out_of_sync if is_out_of_sync_vm?(vm, instance, state)
         return :unbound if is_unbound_instance_vm?(vm, instance, state)
         :ok
       rescue Bosh::Director::RpcTimeout
@@ -103,20 +98,6 @@ module Bosh::Director
       @agent_disks[disk_cid] << vm_cid
     end
 
-    def is_out_of_sync_vm?(vm, instance, state)
-      job = state['job'] ? state['job']['name'] : nil
-      index = state['index']
-      if state['deployment'] != @deployment.name ||
-        (instance && (instance.job != job || instance.index != index))
-        @problem_register.problem_found(:out_of_sync_vm, vm,
-          deployment: state['deployment'],
-          job: job, index: index)
-        true
-      else
-        false
-      end
-    end
-
     def is_unbound_instance_vm?(vm, instance, state)
       job = state['job'] ? state['job']['name'] : nil
       index = state['index']

data/lib/bosh/director/scheduler.rb

@@ -42,16 +42,33 @@ module Bosh::Director
         end
 
         @scheduler.cron(scheduled_job['schedule']) do |_|
-          logger.info("enqueueing `#{scheduled_job['command']}'")
 
-          @queue.enqueue('scheduler',
-                         director_job_class,
-                         "scheduled #{scheduled_job['command']}",
-                         scheduled_job['params'])
+          should_enqueue = true
+          if director_job_class.respond_to?(:has_work)
+            logger.debug("Scheduler cron - checking /
+#{director_job_class}.has_work:#{director_job_class.has_work(scheduled_job['params'])} /
+with params #{scheduled_job['params']}")
+
+            should_enqueue = director_job_class.has_work(scheduled_job['params'])
+          end
+
+          if should_enqueue
+            logger.info("enqueueing `#{scheduled_job['command']}'")
+
+            schedule_message = "scheduled #{scheduled_job['command']}"
+            if director_job_class.respond_to?(:schedule_message)
+              schedule_message = director_job_class.schedule_message
+            end
+
+            @queue.enqueue('scheduler',
+              director_job_class,
+              schedule_message,
+              scheduled_job['params'])
+          end
         end
 
         logger.info("added scheduled job `#{director_job_class}' with interval '#{scheduled_job['schedule']}'")
       end
     end
   end
-end
+end

data/lib/bosh/director/{instance_updater/stopper.rb → stopper.rb}

@@ -1,33 +1,39 @@
 module Bosh::Director
-  class InstanceUpdater::Stopper
-    def initialize(instance, agent_client, target_state, skip_drain, config, logger)
-      @instance = instance
-      @agent_client = agent_client
+  class Stopper
+    def initialize(instance_plan, target_state, config, logger)
+      @instance_plan = instance_plan
+      @instance_model = @instance_plan.new? ? instance_plan.instance.model : instance_plan.existing_instance
       @target_state = target_state
-      @skip_drain = skip_drain
       @config = config
       @logger = logger
     end
 
     def stop
-      if @skip_drain
-        @logger.info("Skipping drain for '#{@instance}'")
+      return if @instance_model.compilation || @instance_model.vm.nil?
+
+      if @instance_plan.skip_drain
+        @logger.info("Skipping drain for '#{@instance_model}'")
       else
         perform_drain
       end
 
-      @agent_client.stop
+      agent_client.stop
     end
 
     private
 
+    def agent_client
+      @agent_client ||= AgentClient.with_vm(@instance_model.vm)
+    end
+
     def perform_drain
-      drain_type = shutting_down? ? 'shutdown' : 'update'
+      drain_type = needs_drain_to_migrate_data? ? 'shutdown' : 'update'
 
       # Apply spec might change after shutdown drain (unlike update drain)
       # because instance's VM could be reconfigured.
       # Drain script can still capture intent from non-final apply spec.
-      drain_time = @agent_client.drain(drain_type, @instance.spec)
+      drain_apply_spec = @instance_plan.spec.as_apply_spec
+      drain_time = agent_client.drain(drain_type, drain_apply_spec)
 
       if drain_time > 0
         sleep(drain_time)
@@ -36,12 +42,12 @@ module Bosh::Director
       end
     end
 
-    def shutting_down?
-      @instance.resource_pool_changed? ||
-        @instance.persistent_disk_changed? ||
-        @instance.networks_changed? ||
-        @target_state == 'stopped' ||
-        @target_state == 'detached'
+    def needs_drain_to_migrate_data?
+      @target_state == 'stopped' ||
+        @target_state == 'detached' ||
+        @instance_plan.needs_shutting_down? ||
+        @instance_plan.persistent_disk_changed? ||
+        @instance_plan.networks_changed?
     end
 
     def wait_for_dynamic_drain(initial_drain_time)
@@ -50,7 +56,7 @@ module Bosh::Director
       loop do
         wait_time = drain_time.abs
         if wait_time > 0
-          @logger.info("`#{@instance}' is draining: checking back in #{wait_time}s")
+          @logger.info("`#{@instance_model}' is draining: checking back in #{wait_time}s")
           sleep(wait_time)
         end
 
@@ -62,7 +68,7 @@ module Bosh::Director
         # realistically speaking, all agents have already been updated
         # to support drain status mechanism and swallowing real errors
         # would be bad here, as it could mask potential problems.
-        drain_time = @agent_client.drain('status')
+        drain_time = agent_client.drain('status')
       end
     end
   end

data/lib/bosh/director/tagged_logger.rb

@@ -0,0 +1,30 @@
+module Bosh::Director
+  class TaggedLogger
+    def initialize(logger, *tags)
+      @logger = logger
+      @tags = tags.map { |t| "[#{t}]" }.join('')
+    end
+
+    def error(message)
+      @logger.error(tag_message(message))
+    end
+
+    def info(message)
+      @logger.info(tag_message(message))
+    end
+
+    def debug(message)
+      @logger.debug(tag_message(message))
+    end
+
+    def warn(message)
+      @logger.warn(tag_message(message))
+    end
+
+    private
+
+    def tag_message(message)
+      "#{@tags} #{message}"
+    end
+  end
+end

data/lib/bosh/director/transactor.rb

@@ -0,0 +1,9 @@
+module Bosh::Director
+  class Transactor
+    def retryable_transaction(db, &block)
+      Bosh::Common.retryable(tries: 3, on: [Sequel::DatabaseError], matching: /Mysql2::Error: Deadlock found when trying to get lock/) do |attempt, e|
+        db.transaction(&block) || true
+      end
+    end
+  end
+end

data/lib/bosh/director/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Director
-    VERSION = '1.3160.0'
+    VERSION = '1.3163.0'
   end
 end

data/lib/bosh/director/vm_creator.rb

@@ -5,17 +5,95 @@ module Bosh::Director
   class VmCreator
     include EncryptionHelper
 
-    def self.create(*args)
-      new.create(*args)
+    def initialize(cloud, logger, vm_deleter, disk_manager, job_renderer)
+      @cloud = cloud
+      @logger = logger
+      @vm_deleter = vm_deleter
+      @disk_manager = disk_manager
+      @job_renderer = job_renderer
     end
 
-    def initialize
-      @cloud = Config.cloud
-      @logger = Config.logger
+    def create_for_instance_plans(instance_plans, ip_provider, event_log)
+      return @logger.info('No missing vms to create') if instance_plans.empty?
+
+      total = instance_plans.size
+      event_log.begin_stage('Creating missing vms', total)
+      ThreadPool.new(max_threads: Config.max_threads, logger: @logger).wrap do |pool|
+        instance_plans.each do |instance_plan|
+          instance = instance_plan.instance
+
+          pool.process do
+            with_thread_name("create_missing_vm(#{instance}/#{total})") do
+              event_log.track(instance.model.to_s) do
+                @logger.info('Creating missing VM')
+                disks = [instance.model.persistent_disk_cid].compact
+                create_for_instance_plan(instance_plan, disks)
+
+                instance_plan.network_plans
+                  .select(&:obsolete?)
+                  .each do |network_plan|
+                  reservation = network_plan.reservation
+                  ip_provider.release(reservation)
+                end
+                instance_plan.release_obsolete_network_plans
+              end
+            end
+          end
+        end
+      end
     end
 
-    def create(deployment, stemcell, cloud_properties, network_settings,
-               disks=nil, env={})
+    def create_for_instance_plan(instance_plan, disks)
+      instance = instance_plan.instance
+      @logger.info('Creating VM')
+
+      vm_model = create(
+        instance.deployment_model,
+        instance.stemcell,
+        instance.cloud_properties,
+        instance_plan.network_settings_hash,
+        disks,
+        instance.env,
+      )
+
+      begin
+        instance.bind_to_vm_model(vm_model)
+        VmMetadataUpdater.build.update(vm_model, {})
+
+        agent_client = AgentClient.with_vm(vm_model)
+        agent_client.wait_until_ready
+        instance.update_trusted_certs
+        instance.update_cloud_properties!
+      rescue Exception => e
+        @logger.error("Failed to create/contact VM #{vm_model.cid}: #{e.inspect}")
+        if Config.keep_unreachable_vms
+          @logger.info("Keeping the VM for debugging")
+        else
+          @vm_deleter.delete_for_instance_plan(instance_plan)
+        end
+        raise e
+      end
+
+      @disk_manager.attach_disks_if_needed(instance_plan)
+
+      apply_initial_vm_state(instance_plan)
+
+      instance_plan.mark_desired_network_plans_as_existing
+    end
+
+    private
+
+    def apply_initial_vm_state(instance_plan)
+      instance_plan.instance.apply_initial_vm_state(instance_plan.spec)
+
+      unless instance_plan.instance.compilation?
+        # re-render job templates with updated dynamic network settings
+        @logger.debug("Re-rendering templates with spec: #{instance_plan.spec.as_template_spec}")
+        @job_renderer.render_job_instance(instance_plan)
+      end
+    end
+
+    def create(deployment, stemcell, cloud_properties, network_settings, disks, env)
       vm = nil
       vm_cid = nil
 
@@ -24,9 +102,9 @@ module Bosh::Director
       agent_id = self.class.generate_agent_id
 
       options = {
-          :deployment => deployment,
-          :agent_id => agent_id,
-          :env => env
+        :deployment => deployment,
+        :agent_id => agent_id,
+        :env => env
       }
 
       if Config.encryption?
@@ -41,7 +119,7 @@ module Bosh::Director
         vm_cid = @cloud.create_vm(agent_id, stemcell.cid, cloud_properties, network_settings, disks, env)
       rescue Bosh::Clouds::VMCreationFailed => e
         count += 1
-        logger.error("failed to create VM, retrying (#{count})")
+        @logger.error("failed to create VM, retrying (#{count})")
         retry if e.ok_to_retry && count < Config.max_vm_create_tries
         raise e
       end
@@ -50,10 +128,9 @@ module Bosh::Director
       vm = Models::Vm.new(options)
 
       vm.save
-      VmMetadataUpdater.build.update(vm, {})
       vm
     rescue => e
-      logger.error("error creating vm: #{e.message}")
+      @logger.error("error creating vm: #{e.message}")
       delete_vm(vm_cid) if vm_cid
       vm.destroy if vm
       raise e
@@ -62,16 +139,11 @@ module Bosh::Director
     def delete_vm(vm_cid)
       @cloud.delete_vm(vm_cid)
     rescue => e
-      logger.error("error cleaning up #{vm_cid}: #{e.message}\n#{e.backtrace.join("\n")}")
+      @logger.error("error cleaning up #{vm_cid}: #{e.message}\n#{e.backtrace.join("\n")}")
     end
 
     def self.generate_agent_id
       SecureRandom.uuid
     end
-
-    def logger
-      Config.logger
-    end
-
   end
 end