bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/instance_updater/preparer.rb

@@ -1,16 +1,19 @@
 module Bosh::Director
   class InstanceUpdater::Preparer
-    def initialize(instance, agent_client, logger)
-      @instance = instance
+    def initialize(instance_plan, agent_client, logger)
+      @instance_plan = instance_plan
       @agent_client = agent_client
       @logger = logger
     end
 
     def prepare
-      # If resource pool has changed or instance will be detached
+      instance = @instance_plan.instance
+      # If resource pool has changed or instance will be recreated/detached
       # there is no point in preparing current VM for future since it will be destroyed.
-      if !@instance.resource_pool_changed? && @instance.state != 'detached'
-        @agent_client.prepare(@instance.spec)
+      unless @instance_plan.needs_shutting_down? || instance.state == 'detached'
+
+        apply_spec = DeploymentPlan::InstanceSpec.create_from_instance_plan(@instance_plan).as_apply_spec
+        @agent_client.prepare(apply_spec)
       end
     rescue RpcRemoteException => e
       if e.message =~ /unknown message/

data/lib/bosh/director/instance_updater/state_applier.rb

@@ -0,0 +1,21 @@
+module Bosh::Director
+  class InstanceUpdater::StateApplier
+    def initialize(instance_plan, agent_client, rendered_job_templates_cleaner)
+      @instance_plan = instance_plan
+      @instance = @instance_plan.instance
+      @agent_client = agent_client
+      @rendered_job_templates_cleaner = rendered_job_templates_cleaner
+    end
+
+    def apply
+      @instance.apply_vm_state(@instance_plan.spec)
+      @instance.update_templates(@instance_plan.templates)
+      @rendered_job_templates_cleaner.clean
+
+      if @instance.state == 'started'
+        @agent_client.run_script('pre-start', {})
+        @agent_client.start
+      end
+    end
+  end
+end

data/lib/bosh/director/ip_util.rb

@@ -1,5 +1,3 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director
   module IpUtil
 
@@ -17,29 +15,6 @@ module Bosh::Director
       end
     end
 
-    def process_range(range)
-      parts = range.split("-")
-      parts.each { |part| part.strip! }
-      if parts.size == 1
-        range    = NetAddr::CIDR.create(parts[0])
-        first_ip = range.first(:Objectify => true).to_i
-        last_ip  = range.last(:Objectify => true).to_i
-        (first_ip .. last_ip).each do |ip|
-          yield ip
-        end
-      elsif parts.size == 2
-        first_ip = NetAddr::CIDR.create(parts[0])
-        last_ip  = NetAddr::CIDR.create(parts[1])
-        raise ArgumentError unless first_ip.size == 1
-        raise ArgumentError unless last_ip.size == 1
-        (first_ip.to_i .. last_ip.to_i).each do |ip|
-          yield ip
-        end
-      else
-        raise ArgumentError
-      end
-    end
-
     def ip_to_i(ip)
       unless ip.kind_of?(Integer)
         unless ip.kind_of?(NetAddr::CIDR)
@@ -63,5 +38,50 @@ module Bosh::Director
       ip_to_netaddr(ip).ip
     end
 
+    private
+
+    def process_range(range)
+      parts = range.split("-")
+      parts.each { |part| part.strip! }
+      if parts.size == 1
+        range = NetAddr::CIDR.create(parts[0])
+        first_ip = range.first(:Objectify => true).to_i
+        last_ip = range.last(:Objectify => true).to_i
+        (first_ip .. last_ip).each do |ip|
+          yield ip
+        end
+      elsif parts.size == 2
+        first_ip = NetAddr::CIDR.create(parts[0])
+        last_ip = NetAddr::CIDR.create(parts[1])
+        unless first_ip.size == 1 && last_ip.size == 1
+          raise NetworkInvalidIpRangeFormat, "Invalid IP range format: #{range}"
+        end
+        (first_ip.to_i .. last_ip.to_i).each do |ip|
+          yield ip
+        end
+      else
+        raise NetworkInvalidIpRangeFormat,
+          "Invalid IP range format: #{range}"
+      end
+    rescue ArgumentError, NetAddr::ValidationError => e
+      raise NetworkInvalidIpRangeFormat, e.message
+    end
+    class CIDRIP
+      def initialize(ip)
+        if ip.kind_of?(NetAddr::CIDR)
+          @cidr = ip
+        else
+          @cidr = NetAddr::CIDR.create(ip)
+        end
+      end
+
+      def to_i
+        @cidr.to_i
+      end
+
+      def to_s
+        @cidr.ip.to_s
+      end
+    end
   end
-end
+end

data/lib/bosh/director/job_renderer.rb

@@ -3,21 +3,33 @@ require 'bosh/director/core/templates/job_instance_renderer'
 
 module Bosh::Director
   class JobRenderer
-    # @param [DeploymentPlan::Job]
-    def initialize(job, blobstore)
-      @job = job
-      @blobstore = blobstore
+    def self.create
+      new(App.instance.blobstores.blobstore, Config.logger)
+    end
 
-      job_template_loader = Core::Templates::JobTemplateLoader.new(Config.logger)
-      @instance_renderer = Core::Templates::JobInstanceRenderer.new(@job.templates, job_template_loader)
+    def initialize(blobstore, logger)
+      @blobstore = blobstore
+      @logger = logger
     end
 
-    def render_job_instances
-      @job.instances.each { |instance| render_job_instance(instance) }
+    def render_job_instances(instance_plan)
+      instance_plan.each { |instance_plan| render_job_instance(instance_plan) }
     end
 
-    def render_job_instance(instance)
-      rendered_job_instance = @instance_renderer.render(instance.spec)
+    def render_job_instance(instance_plan)
+      instance = instance_plan.instance
+
+      if instance_plan.templates.empty?
+        @logger.debug("Skipping rendering templates for '#{instance}', no templates")
+        return
+      end
+
+      @logger.debug("Rendering templates for instance #{instance}")
+
+      job_template_loader = Core::Templates::JobTemplateLoader.new(@logger)
+
+      instance_renderer = Core::Templates::JobInstanceRenderer.new(instance_plan.templates, job_template_loader)
+      rendered_job_instance = instance_renderer.render(instance_plan.spec.as_template_spec)
 
       configuration_hash = rendered_job_instance.configuration_hash
 

data/lib/bosh/director/job_runner.rb

@@ -67,10 +67,7 @@ module Bosh::Director
       Config.redis_logger = redis_task_logger
 
       Config.db.logger = @task_logger
-
-      if Config.dns_enabled?
-        Config.dns_db.logger = @task_logger
-      end
+      Config.dns_db.logger = @task_logger if Config.dns_db
 
       cpi_log = File.join(log_dir, 'cpi')
       Config.cloud_options['properties'] ||= {}

data/lib/bosh/director/job_updater.rb

@@ -3,75 +3,88 @@ module Bosh::Director
     # @param [Bosh::Director::DeploymentPlan::Planner] deployment_plan
     # @param [Bosh::Director::DeploymentPlan::Job] job
     # @param [Bosh::Director::JobRenderer] job_renderer
-    def initialize(deployment_plan, job, job_renderer)
+    def initialize(deployment_plan, job, links_resolver, disk_manager)
       @deployment_plan = deployment_plan
       @job = job
-      @job_renderer = job_renderer
+      @links_resolver = links_resolver
 
       @logger = Config.logger
       @event_log = Config.event_log
+      @disk_manager = disk_manager
     end
 
     def update
       @logger.info('Deleting no longer needed instances')
       delete_unneeded_instances
 
-      instances = []
-      @job.instances.each do |instance|
-        instances << instance if instance.changed?
+      instance_plans = @job.needed_instance_plans.select do | instance_plan |
+        if instance_plan.changed?
+          true
+        else
+          # no changes necessary for the agent, but some metadata may have
+          # changed (i.e. vm_type.name), so push state to the db regardless
+          instance_plan.persist_current_spec
+          false
+        end
       end
 
-      if instances.empty?
-        @logger.info("No instances to update for `#{@job.name}'")
+      if instance_plans.empty?
+        @logger.info("No instances to update for '#{@job.name}'")
         return
       end
 
-      @logger.info("Found #{instances.size} instances to update")
-      event_log_stage = @event_log.begin_stage("Updating job", instances.size, [ @job.name ])
+      instance_plans.each do |instance_plan|
+        changes = instance_plan.changes
+        @logger.debug("Need to update instance '#{instance_plan.instance}', changes: #{changes.to_a.join(', ').inspect}")
+      end
+
+      @logger.info("Found #{instance_plans.size} instances to update")
+      event_log_stage = @event_log.begin_stage('Updating job', instance_plans.size, [ @job.name ])
 
       ThreadPool.new(:max_threads => @job.update.max_in_flight).wrap do |pool|
-        num_canaries = [ @job.update.canaries, instances.size ].min
+        num_canaries = [ @job.update.canaries, instance_plans.size ].min
         @logger.info("Starting canary update num_canaries=#{num_canaries}")
-        update_canaries(pool, instances, num_canaries, event_log_stage)
+        update_canaries(pool, instance_plans, num_canaries, event_log_stage)
 
         @logger.info('Waiting for canaries to update')
         pool.wait
 
-        @logger.info("Finished canary update")
+        @logger.info('Finished canary update')
 
-        @logger.info("Continuing the rest of the update")
-        update_instances(pool, instances, event_log_stage)
+        @logger.info('Continuing the rest of the update')
+        update_instances(pool, instance_plans, event_log_stage)
       end
 
-      @logger.info("Finished the rest of the update")
+      @logger.info('Finished the rest of the update')
     end
 
     private
 
     def delete_unneeded_instances
+      unneeded_instance_plans = @job.obsolete_instance_plans
       unneeded_instances = @job.unneeded_instances
       return if unneeded_instances.empty?
 
-      event_log_stage = @event_log.begin_stage("Deleting unneeded instances", unneeded_instances.size, [@job.name])
-      deleter = InstanceDeleter.new(@deployment_plan)
-      deleter.delete_instances(unneeded_instances, event_log_stage, max_threads: @job.update.max_in_flight)
+      event_log_stage = @event_log.begin_stage('Deleting unneeded instances', unneeded_instances.size, [@job.name])
+      dns_manager = DnsManager.create
+      deleter = InstanceDeleter.new(@deployment_plan.ip_provider, dns_manager, @disk_manager)
+      deleter.delete_instance_plans(unneeded_instance_plans, event_log_stage, max_threads: @job.update.max_in_flight)
 
-      @logger.info("Deleted no longer needed instances")
+      @logger.info('Deleted no longer needed instances')
     end
 
-    def update_canaries(pool, instances, num_canaries, event_log_stage)
+    def update_canaries(pool, instance_plans, num_canaries, event_log_stage)
       num_canaries.times do
-        instance = instances.shift
-        pool.process { update_canary_instance(instance, event_log_stage) }
+        instance_plan = instance_plans.shift
+        pool.process { update_canary_instance(instance_plan, event_log_stage) }
       end
     end
 
-    def update_canary_instance(instance, event_log_stage)
-      desc = "#{@job.name}/#{instance.index}"
-      event_log_stage.advance_and_track("#{desc} (canary)") do |ticker|
-        with_thread_name("canary_update(#{desc})") do
+    def update_canary_instance(instance_plan, event_log_stage)
+      event_log_stage.advance_and_track("#{instance_plan.instance.model} (canary)") do
+        with_thread_name("canary_update(#{instance_plan.instance.model})") do
           begin
-            InstanceUpdater.new(instance, ticker, @job_renderer).update(:canary => true)
+            InstanceUpdater.new_instance_updater(@deployment_plan.ip_provider).update(instance_plan, :canary => true)
           rescue Exception => e
             @logger.error("Error updating canary instance: #{e.inspect}\n#{e.backtrace.join("\n")}")
             raise
@@ -80,18 +93,17 @@ module Bosh::Director
       end
     end
 
-    def update_instances(pool, instances, event_log_stage)
-      instances.each do |instance|
-        pool.process { update_instance(instance, event_log_stage) }
+    def update_instances(pool, instance_plans, event_log_stage)
+      instance_plans.each do |instance_plan|
+        pool.process { update_instance(instance_plan, event_log_stage) }
       end
     end
 
-    def update_instance(instance, event_log_stage)
-      desc = "#{@job.name}/#{instance.index}"
-      event_log_stage.advance_and_track(desc) do |ticker|
-        with_thread_name("instance_update(#{desc})") do
+    def update_instance(instance_plan, event_log_stage)
+      event_log_stage.advance_and_track("#{instance_plan.instance.model}") do
+        with_thread_name("instance_update(#{instance_plan.instance.model})") do
           begin
-            InstanceUpdater.new(instance, ticker, @job_renderer).update
+            InstanceUpdater.new_instance_updater(@deployment_plan.ip_provider).update(instance_plan)
           rescue Exception => e
             @logger.error("Error updating instance: #{e.inspect}\n#{e.backtrace.join("\n")}")
             raise

data/lib/bosh/director/job_updater_factory.rb

@@ -1,12 +1,13 @@
 module Bosh::Director
   class JobUpdaterFactory
-    def initialize(blobstore)
-      @blobstore = blobstore
+    def initialize(cloud, logger)
+      @cloud = cloud
+      @logger = logger
     end
 
     def new_job_updater(deployment_plan, job)
-      job_renderer = JobRenderer.new(job, @blobstore)
-      JobUpdater.new(deployment_plan, job, job_renderer)
+      links_resolver = DeploymentPlan::LinksResolver.new(deployment_plan, @logger)
+      JobUpdater.new(deployment_plan, job, links_resolver, DiskManager.new(@cloud, @logger))
     end
   end
 end

data/lib/bosh/director/jobs/base_job.rb

@@ -10,6 +10,10 @@ module Bosh::Director
         Bosh::Director::JobRunner.new(self, task_id).run(*args)
       end
 
+      def self.schedule_message
+        "scheduled #{name.split('::').last}"
+      end
+
       attr_accessor :task_id
 
       def logger
@@ -24,6 +28,10 @@ module Bosh::Director
         @result_file ||= Config.result
       end
 
+      def dns_manager
+        @dns_manager ||= DnsManager.create
+      end
+
       # @return [Boolean] Has task been cancelled?
       def task_cancelled?
         return false if task_id.nil?

data/lib/bosh/director/jobs/cleanup_artifacts.rb

@@ -0,0 +1,93 @@
+module Bosh::Director
+  module Jobs
+    class CleanupArtifacts < BaseJob
+      include Bosh::Director::LockHelper
+
+      @queue = :normal
+
+      def self.job_type
+        :delete_artifacts
+      end
+
+      def self.enqueue(username, config, job_queue)
+        description = config['remove_all'] ? 'clean up all' : 'clean up'
+        job_queue.enqueue(username, Jobs::CleanupArtifacts, description, [config])
+      end
+
+      def initialize(config)
+        @config = config
+        @disk_manager = DiskManager.new(Config.cloud, Config.logger)
+        release_manager = Api::ReleaseManager.new
+        @stemcell_manager = Api::StemcellManager.new
+        blobstore = App.instance.blobstores.blobstore
+        cloud = Config.cloud
+        blob_deleter = Jobs::Helpers::BlobDeleter.new(blobstore, logger)
+        compiled_package_deleter = Jobs::Helpers::CompiledPackageDeleter.new(blob_deleter, logger)
+        @stemcell_deleter = Jobs::Helpers::StemcellDeleter.new(cloud, compiled_package_deleter, logger, event_log)
+        @releases_to_delete_picker = Jobs::Helpers::ReleasesToDeletePicker.new(release_manager)
+        @stemcells_to_delete_picker = Jobs::Helpers::StemcellsToDeletePicker.new(@stemcell_manager)
+        package_deleter = Helpers::PackageDeleter.new(compiled_package_deleter, blob_deleter, logger)
+        template_deleter = Helpers::TemplateDeleter.new(blob_deleter, logger)
+        release_deleter = Helpers::ReleaseDeleter.new(package_deleter, template_deleter, event_log, logger)
+        release_version_deleter =
+          Helpers::ReleaseVersionDeleter.new(release_deleter, package_deleter, template_deleter, logger, event_log)
+        @name_version_release_deleter =
+          Helpers::NameVersionReleaseDeleter.new(release_deleter, release_manager, release_version_deleter, logger)
+      end
+
+      def perform
+        if @config['remove_all']
+          releases_to_keep, stemcells_to_keep = 0, 0
+        else
+          releases_to_keep, stemcells_to_keep = 2, 2
+        end
+
+        unused_release_name_and_version = @releases_to_delete_picker.pick(releases_to_keep)
+        release_stage = event_log.begin_stage('Deleting releases', unused_release_name_and_version.count)
+        ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
+          unused_release_name_and_version.each do |name_and_version|
+            pool.process do
+              name = name_and_version['name']
+              version = name_and_version['version']
+              release_stage.advance_and_track("#{name}/#{version}") do
+                with_release_lock(name, :timeout => 10) do
+                  @name_version_release_deleter.find_and_delete_release(name, version, false)
+                end
+              end
+            end
+          end
+        end
+
+        stemcells_to_delete = @stemcells_to_delete_picker.pick(stemcells_to_keep)
+        stemcell_stage = event_log.begin_stage('Deleting stemcells', stemcells_to_delete.count)
+        ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
+          stemcells_to_delete.each do |stemcell|
+            pool.process do
+              stemcell_stage.advance_and_track("#{stemcell['name']}/#{stemcell['version']}") do
+                stemcell_to_delete = @stemcell_manager.find_by_name_and_version(stemcell['name'], stemcell['version'])
+                @stemcell_deleter.delete(stemcell_to_delete)
+              end
+            end
+          end
+        end
+
+        orphan_disks = []
+        if @config['remove_all']
+          orphan_disks = Models::OrphanDisk.all
+          orphan_disk_stage = event_log.begin_stage('Deleting orphaned disks', orphan_disks.count)
+          ThreadPool.new(:max_threads => Config.max_threads).wrap do |pool|
+            orphan_disks.each do |orphan_disk|
+              pool.process do
+                orphan_disk_stage.advance_and_track("#{orphan_disk.disk_cid}") do
+                  @disk_manager.delete_orphan_disk(orphan_disk)
+                end
+              end
+            end
+          end
+        end
+
+        "Deleted #{unused_release_name_and_version.count} release(s), #{stemcells_to_delete.count} stemcell(s), #{orphan_disks.count} orphaned disk(s)"
+      end
+    end
+  end
+end