bosh-director 1.2583.0 → 1.2596.0

data/lib/bosh/director/instance_updater/network_updater.rb

@@ -1,10 +1,10 @@
 module Bosh::Director
   class InstanceUpdater::NetworkUpdater
-    def initialize(instance, vm_model, agent_client, resource_pool_updater, cloud, logger)
+    def initialize(instance, vm_model, agent_client, vm_updater, cloud, logger)
       @instance = instance
       @vm_model = vm_model
       @agent_client = agent_client
-      @resource_pool_updater = resource_pool_updater
+      @vm_updater = vm_updater
       @cloud = cloud
       @logger = logger
     end
@@ -12,40 +12,38 @@ module Bosh::Director
     def update
       unless @instance.networks_changed?
         @logger.info('Skipping network re-configuration')
-        return
+        return [@vm_model, @agent_client]
       end
 
       network_settings = @instance.network_settings
-      @logger.info("Planning to reconfigure network with settings: #{network_settings}")
 
       strategies = [
         ConfigureNetworksStrategy.new(@agent_client, network_settings, @logger),
         PrepareNetworkChangeStrategy.new(@agent_client, network_settings, @logger),
       ]
 
+      @logger.info("Planning to reconfigure network with settings: #{network_settings}")
       selected_strategy = strategies.find { |s| s.before_configure_networks }
 
       @cloud.configure_networks(@vm_model.cid, network_settings)
 
       selected_strategy.after_configure_networks
 
-    rescue Bosh::Clouds::NotSupported => e
-      # If configure_networks can't configure the network as
-      # requested, e.g. when the security groups change on AWS,
-      # configure_networks() will raise an exception and we'll
-      # recreate the VM to work around it
-      @logger.info("configure_networks CPI call failed with error: #{e.inspect}")
-      configure_new_vm
-    end
+      [@vm_model, @agent_client]
 
-    private
+    rescue Bosh::Clouds::NotSupported => e
+      @logger.info("Failed reconfiguring existing VM: #{e.inspect}")
 
-    def configure_new_vm
+      # If configure_networks CPI method cannot reconfigure VM networking
+      # (e.g. when the security groups change on AWS)
+      # it raises Bosh::Clouds::NotSupported to indicate new VM is needed.
       @logger.info('Creating VM with new network configurations')
       @instance.recreate = true
-      @resource_pool_updater.update_resource_pool
+      @vm_updater.update(nil)
     end
 
+    private
+
     # Newer agents support prepare_configure_networks/configure_networks messages
     class ConfigureNetworksStrategy
       def initialize(agent_client, network_settings, logger)

data/lib/bosh/director/instance_updater/vm_updater.rb

@@ -0,0 +1,223 @@
+module Bosh::Director
+  class InstanceUpdater::VmUpdater
+    def initialize(instance, vm_model, agent_client, job_renderer, cloud, max_update_tries, logger)
+      @instance = instance
+      @vm_model = vm_model
+      @agent_client = agent_client
+      @job_renderer = job_renderer
+      @cloud = cloud
+      @max_update_tries = max_update_tries
+      @logger = logger
+    end
+
+    def update(new_disk_cid)
+      unless @instance.resource_pool_changed? || new_disk_cid
+        @logger.info('Skipping VM update')
+        return [@vm_model, @agent_client]
+      end
+
+      disk_detacher = DiskDetacher.new(@instance, @vm_model, @agent_client, @cloud, @logger)
+      disk_detacher.detach
+
+      @max_update_tries.times do |try|
+        vm_deleter = VmDeleter.new(@instance, @vm_model, @cloud, @logger)
+        vm_deleter.delete
+
+        vm_creator = VmCreator.new(@instance, @cloud, @logger)
+        @vm_model, @agent_client = vm_creator.create(new_disk_cid)
+
+        begin
+          # Could raise Bosh::Clouds::NoDiskSpace because some CPIs might lazily create disks
+          disk_attacher = DiskAttacher.new(@instance, @vm_model, @agent_client, @cloud, @logger)
+          disk_attacher.attach
+          break
+        rescue Bosh::Clouds::NoDiskSpace => e
+          if e.ok_to_retry && try < @max_update_tries-1
+            @logger.warn("Retrying attach disk operation #{try}: #{e.inspect}")
+          else
+            @logger.warn("Failed to attach disk to new VM: #{e.inspect}")
+            raise CloudNotEnoughDiskSpace,
+                  "Not enough disk space to update `#{@instance}'"
+          end
+        end
+      end
+
+      vm_state_applier = VmStateApplier.new(@instance, @vm_model, @agent_client, @job_renderer, @logger)
+      vm_state_applier.apply
+
+      [@vm_model, @agent_client]
+    end
+
+    def detach
+      @logger.info('Detaching VM')
+
+      disk_detacher = DiskDetacher.new(@instance, @vm_model, @agent_client, @cloud, @logger)
+      disk_detacher.detach
+
+      vm_deleter = VmDeleter.new(@instance, @vm_model, @cloud, @logger)
+      vm_deleter.delete
+
+      @instance.job.resource_pool.add_idle_vm
+    end
+
+    def attach_missing_disk
+      if !@instance.model.persistent_disk_cid || @instance.disk_currently_attached?
+        @logger.info('Skipping attaching missing VM')
+        return
+      end
+
+      begin
+        disk_attacher = DiskAttacher.new(@instance, @vm_model, @agent_client, @cloud, @logger)
+        disk_attacher.attach
+      rescue Bosh::Clouds::NoDiskSpace => e
+        @logger.warn("Failed attaching missing disk first time: #{e.inspect}")
+        update(@instance.model.persistent_disk_cid)
+      end
+    end
+
+    private
+
+    class VmCreator
+      def initialize(instance, cloud, logger)
+        @instance = instance
+        @cloud = cloud
+        @logger = logger
+      end
+
+      def create(new_disk_id)
+        @logger.info('Creating VM')
+
+        deployment = @instance.job.deployment
+        resource_pool = @instance.job.resource_pool
+
+        vm_model = Bosh::Director::VmCreator.create(
+          deployment.model,
+          resource_pool.stemcell.model,
+          resource_pool.cloud_properties,
+          @instance.network_settings,
+          [@instance.model.persistent_disk_cid, new_disk_id].compact,
+          resource_pool.env,
+        )
+
+        begin
+          @instance.model.vm = vm_model
+          @instance.model.save
+
+          agent_client = AgentClient.with_defaults(vm_model.agent_id)
+          agent_client.wait_until_ready
+        rescue Exception => e
+          @logger.error("Failed to create/contact VM #{vm_model.cid}: #{e.inspect}")
+          VmDeleter.new(@instance, vm_model, @cloud, @logger).delete
+          raise e
+        end
+
+        [vm_model, agent_client]
+      end
+    end
+
+    class VmDeleter
+      def initialize(instance, vm_model, cloud, logger)
+        @instance = instance
+        @vm_model = vm_model
+        @cloud = cloud
+        @logger = logger
+      end
+
+      def delete
+        @logger.info('Deleting VM')
+
+        @cloud.delete_vm(@vm_model.cid)
+
+        @instance.model.db.transaction do
+          @instance.model.vm = nil
+          @instance.model.save
+
+          @vm_model.destroy
+        end
+      end
+    end
+
+    class DiskAttacher
+      def initialize(instance, vm_model, agent_client, cloud, logger)
+        @instance = instance
+        @vm_model = vm_model
+        @agent_client = agent_client
+        @cloud = cloud
+        @logger = logger
+      end
+
+      def attach
+        if @instance.model.persistent_disk_cid.nil?
+          @logger.info('Skipping disk attaching')
+          return
+        end
+
+        @cloud.attach_disk(@vm_model.cid, @instance.model.persistent_disk_cid)
+
+        @agent_client.mount_disk(@instance.model.persistent_disk_cid)
+      end
+    end
+
+    class DiskDetacher
+      def initialize(instance, vm_model, agent_client, cloud, logger)
+        @instance = instance
+        @vm_model = vm_model
+        @agent_client = agent_client
+        @cloud = cloud
+        @logger = logger
+      end
+
+      def detach
+        unless @instance.disk_currently_attached?
+          @logger.info('Skipping disk detaching')
+          return
+        end
+
+        if @instance.model.persistent_disk_cid.nil?
+          raise AgentUnexpectedDisk,
+                "`#{@instance}' VM has disk attached but it's not reflected in director DB"
+        end
+
+        @agent_client.unmount_disk(@instance.model.persistent_disk_cid)
+
+        @cloud.detach_disk(@vm_model.cid, @instance.model.persistent_disk_cid)
+      end
+    end
+
+    class VmStateApplier
+      def initialize(instance, vm_model, agent_client, job_renderer, logger)
+        @instance = instance
+        @vm_model = vm_model
+        @agent_client = agent_client
+        @job_renderer = job_renderer
+        @logger = logger
+      end
+
+      def apply
+        @logger.info('Applying VM state')
+
+        state = {
+          'deployment' => @instance.job.deployment.name,
+          'networks' => @instance.network_settings,
+          'resource_pool' => @instance.job.resource_pool.spec,
+          'job' => @instance.job.spec,
+          'index' => @instance.index,
+        }
+
+        if @instance.disk_size > 0
+          state['persistent_disk'] = @instance.disk_size
+        end
+
+        @vm_model.update(:apply_spec => state)
+
+        @agent_client.apply(state)
+
+        # Agent will potentially return modified version of state
+        # with resolved dynamic networks information
+        @instance.current_state = @agent_client.get_state
+
+        @job_renderer.render_job_instance(@instance)
+      end
+    end
+  end
+end

data/lib/bosh/director/job_renderer.rb

@@ -4,35 +4,43 @@ require 'bosh/director/core/templates/job_instance_renderer'
 module Bosh::Director
   class JobRenderer
     # @param [DeploymentPlan::Job]
-    def initialize(job)
+    def initialize(job, blobstore)
       @job = job
+      @blobstore = blobstore
+
       job_template_loader = Core::Templates::JobTemplateLoader.new(Config.logger)
       @instance_renderer = Core::Templates::JobInstanceRenderer.new(@job.templates, job_template_loader)
     end
 
-    def render_job_instances(blobstore)
-      @job.instances.each do |instance|
-        rendered_job_instance = @instance_renderer.render(instance.spec)
-
-        configuration_hash = rendered_job_instance.configuration_hash
-
-        archive_model = instance.model.latest_rendered_templates_archive
-        if archive_model && archive_model.content_sha1 == configuration_hash
-          rendered_templates_archive = Core::Templates::RenderedTemplatesArchive.new(archive_model.blobstore_id, archive_model.sha1)
-        else
-          rendered_templates_archive = rendered_job_instance.persist(blobstore)
-          instance.model.add_rendered_templates_archive(
-            blobstore_id: rendered_templates_archive.blobstore_id,
-            sha1: rendered_templates_archive.sha1,
-            content_sha1: configuration_hash,
-            created_at: Time.now,
-          )
-        end
-
-        instance.configuration_hash = configuration_hash
-        instance.template_hashes    = rendered_job_instance.template_hashes
-        instance.rendered_templates_archive = rendered_templates_archive
+    def render_job_instances
+      @job.instances.each { |instance| render_job_instance(instance) }
+    end
+
+    def render_job_instance(instance)
+      rendered_job_instance = @instance_renderer.render(instance.spec)
+
+      configuration_hash = rendered_job_instance.configuration_hash
+
+      archive_model = instance.model.latest_rendered_templates_archive
+
+      if archive_model && archive_model.content_sha1 == configuration_hash
+        rendered_templates_archive = Core::Templates::RenderedTemplatesArchive.new(
+          archive_model.blobstore_id,
+          archive_model.sha1,
+        )
+      else
+        rendered_templates_archive = rendered_job_instance.persist(@blobstore)
+        instance.model.add_rendered_templates_archive(
+          blobstore_id: rendered_templates_archive.blobstore_id,
+          sha1: rendered_templates_archive.sha1,
+          content_sha1: configuration_hash,
+          created_at: Time.now,
+        )
       end
+
+      instance.configuration_hash = configuration_hash
+      instance.template_hashes    = rendered_job_instance.template_hashes
+      instance.rendered_templates_archive = rendered_templates_archive
     end
   end
 end

data/lib/bosh/director/job_updater.rb

@@ -1,16 +1,19 @@
 module Bosh::Director
   class JobUpdater
-    # @param [Bosh::Director::DeploymentPlan] deployment_plan
-    # @param [DeploymentPlan::Job] job
-    def initialize(deployment_plan, job)
+    # @param [Bosh::Director::DeploymentPlan::Planner] deployment_plan
+    # @param [Bosh::Director::DeploymentPlan::Job] job
+    # @param [Bosh::Director::JobRenderer] job_renderer
+    def initialize(deployment_plan, job, job_renderer)
       @deployment_plan = deployment_plan
       @job = job
+      @job_renderer = job_renderer
+
       @logger = Config.logger
       @event_log = Config.event_log
     end
 
     def update
-      @logger.info("Deleting no longer needed instances")
+      @logger.info('Deleting no longer needed instances')
       delete_unneeded_instances
 
       instances = []
@@ -31,7 +34,7 @@ module Bosh::Director
         @logger.info("Starting canary update num_canaries=#{num_canaries}")
         update_canaries(pool, instances, num_canaries, event_log_stage)
 
-        @logger.info("Waiting for canaries to update")
+        @logger.info('Waiting for canaries to update')
         pool.wait
 
         @logger.info("Finished canary update")
@@ -68,7 +71,7 @@ module Bosh::Director
       event_log_stage.advance_and_track("#{desc} (canary)") do |ticker|
         with_thread_name("canary_update(#{desc})") do
           begin
-            InstanceUpdater.new(instance, ticker).update(:canary => true)
+            InstanceUpdater.new(instance, ticker, @job_renderer).update(:canary => true)
           rescue Exception => e
             @logger.error("Error updating canary instance: #{e.inspect}\n#{e.backtrace.join("\n")}")
             raise
@@ -88,7 +91,7 @@ module Bosh::Director
       event_log_stage.advance_and_track(desc) do |ticker|
         with_thread_name("instance_update(#{desc})") do
           begin
-            InstanceUpdater.new(instance, ticker).update
+            InstanceUpdater.new(instance, ticker, @job_renderer).update
           rescue Exception => e
             @logger.error("Error updating instance: #{e.inspect}\n#{e.backtrace.join("\n")}")
             raise

data/lib/bosh/director/job_updater_factory.rb

@@ -0,0 +1,12 @@
+module Bosh::Director
+  class JobUpdaterFactory
+    def initialize(blobstore)
+      @blobstore = blobstore
+    end
+
+    def new_job_updater(deployment_plan, job)
+      job_renderer = JobRenderer.new(job, @blobstore)
+      JobUpdater.new(deployment_plan, job, job_renderer)
+    end
+  end
+end

data/lib/bosh/director/jobs/update_deployment.rb

@@ -12,6 +12,8 @@ module Bosh::Director
       # @param [String] manifest_file Path to deployment manifest
       # @param [Hash] options Deployment options
       def initialize(manifest_file, options = {})
+        @blobstore = App.instance.blobstores.blobstore
+
         logger.info('Reading deployment manifest')
         @manifest_file = manifest_file
         @manifest = File.read(@manifest_file)
@@ -47,7 +49,8 @@ module Bosh::Director
 
       def update
         resource_pools = DeploymentPlan::ResourcePools.new(event_log, @resource_pool_updaters)
-        multi_job_updater = DeploymentPlan::BatchMultiJobUpdater.new
+        job_updater_factory = JobUpdaterFactory.new(@blobstore)
+        multi_job_updater = DeploymentPlan::BatchMultiJobUpdater.new(job_updater_factory)
         updater = DeploymentPlan::Updater.new(self, event_log, resource_pools, @assembler, @deployment_plan, multi_job_updater)
         updater.update
       end

data/lib/bosh/director/jobs/update_release.rb

@@ -1,6 +1,6 @@
 # Copyright (c) 2009-2012 VMware, Inc.
 
-require 'common/version_number'
+require 'common/version/release_version'
 
 module Bosh::Director
   module Jobs
@@ -106,7 +106,7 @@ module Bosh::Director
         @name = @manifest["name"]
 
         begin
-          @version = Bosh::Common::VersionNumber.parse(@manifest["version"])
+          @version = Bosh::Common::Version::ReleaseVersion.parse(@manifest["version"])
           unless @version == @manifest["version"]
             logger.info("Formatted version '#{@manifest["version"]}' => '#{@version}'")
           end
@@ -170,8 +170,7 @@ module Bosh::Director
         event_log.track("#{@name}/#{@version}") {}
       end
 
-      # Normalizes release manifest, so all names, versions, and checksums
-      # are Strings.
+      # Normalizes release manifest, so all names, versions, and checksums are Strings.
       # @return [void]
       def normalize_manifest
         Bosh::Director.hash_string_vals(@manifest, 'name', 'version')
@@ -219,28 +218,18 @@ module Bosh::Director
         existing_packages = []
 
         @manifest["packages"].each do |package_meta|
-          filter = {:sha1 => package_meta["sha1"]}
-          if package_meta["fingerprint"]
-            filter[:fingerprint] = package_meta["fingerprint"]
-            filter = filter.sql_or
-          end
-
           # Checking whether we might have the same bits somewhere
-          packages = Models::Package.where(filter).all
+          packages = Models::Package.where(fingerprint: package_meta["fingerprint"]).all
 
           if packages.empty?
             new_packages << package_meta
             next
           end
 
-          # We can reuse an existing package as long as it
-          # belongs to the same release and has the same name and version.
           existing_package = packages.find do |package|
             package.release_id == @release_model.id &&
             package.name == package_meta["name"] &&
             package.version == package_meta["version"]
-            # NOT checking dependencies here b/c dependency change would
-            # bump the package version anyway.
           end
 
           if existing_package
@@ -251,6 +240,7 @@ module Bosh::Director
             # of the package blob and create a new db entry for it
             package = packages.first
             package_meta["blobstore_id"] = package.blobstore_id
+            package_meta["sha1"] = package.sha1
             new_packages << package_meta
           end
         end
@@ -284,25 +274,10 @@ module Bosh::Director
       def use_existing_packages(packages)
         return if packages.empty?
 
-        n_packages = packages.size
-        event_log.begin_stage("Processing #{n_packages} existing " +
-                              "package#{n_packages > 1 ? "s" : ""}", 1)
-
-        event_log.track("Verifying checksums") do
-          packages.each do |package, package_meta|
+        single_step_stage("Processing #{packages.size} existing package#{"s" if packages.size > 1}") do
+          packages.each do |package, _|
             package_desc = "#{package.name}/#{package.version}"
-            logger.info("Package `#{package_desc}' already exists, " +
-                        "verifying checksum")
-
-            expected = package.sha1
-            received = package_meta["sha1"]
-
-            if expected != received
-              raise ReleaseExistingPackageHashMismatch,
-                    "`#{package_desc}' checksum mismatch, " +
-                      "expected #{expected} but received #{received}"
-            end
-            logger.info("Package `#{package_desc}' verified")
+            logger.info("Using existing package `#{package_desc}'")
             register_package(package)
           end
         end
@@ -359,8 +334,7 @@ module Bosh::Director
 
       # Finds job template definitions in release manifest and sorts them into
       # two buckets: new and existing job templates, then creates new job
-      # template records in the database and points release version to existing
-      # ones.
+      # template records in the database and points release version to existing ones.
       # @return [void]
       def process_jobs
         logger.info("Checking for new jobs in release")
@@ -369,14 +343,8 @@ module Bosh::Director
         existing_jobs = []
 
         @manifest["jobs"].each do |job_meta|
-          filter = {:sha1 => job_meta["sha1"]}
-          if job_meta["fingerprint"]
-            filter[:fingerprint] = job_meta["fingerprint"]
-            filter = filter.sql_or
-          end
-
           # Checking whether we might have the same bits somewhere
-          jobs = Models::Template.where(filter).all
+          jobs = Models::Template.where(fingerprint: job_meta["fingerprint"]).all
 
           template = jobs.find do |job|
             job.release_id == @release_model.id &&
@@ -454,8 +422,7 @@ module Bosh::Director
             path = File.join(job_dir, "templates", relative_path)
             unless File.file?(path)
               raise JobMissingTemplateFile,
-                    "Missing template file `#{relative_path}' " +
-                    "for job `#{template.name}'"
+                    "Missing template file `#{relative_path}' for job `#{template.name}'"
             end
           end
         end
@@ -507,27 +474,10 @@ module Bosh::Director
       def use_existing_jobs(jobs)
         return if jobs.empty?
 
-        n_jobs = jobs.size
-        event_log.begin_stage("Processing #{n_jobs} existing " +
-                              "job#{n_jobs > 1 ? "s" : ""}", 1)
-
-        event_log.track("Verifying checksums") do
-          jobs.each do |template, job_meta|
+        single_step_stage("Processing #{jobs.size} existing job#{"s" if jobs.size > 1}") do
+          jobs.each do |template, _|
             job_desc = "#{template.name}/#{template.version}"
-
-            logger.info("Job `#{job_desc}' already exists, " +
-                        "verifying checksum")
-
-            expected = template.sha1
-            received = job_meta["sha1"]
-
-            if expected != received
-              raise ReleaseExistingJobHashMismatch,
-                    "`#{job_desc}' checksum mismatch, " +
-                    "expected #{expected} but received #{received}"
-            end
-
-            logger.info("Job `#{job_desc}' verified")
+            logger.info("Using existing job `#{job_desc}'")
             register_template(template)
           end
         end
@@ -545,13 +495,11 @@ module Bosh::Director
       # Returns the next release version (to be used for rebased release)
       # @return [String]
       def next_release_version
-        attrs = {
-          :release_id => @release_model.id
-        }
+        attrs = {:release_id => @release_model.id}
         models = Models::ReleaseVersion.filter(attrs).all
         strings = models.map(&:version)
-        versions = Bosh::Common::VersionNumber.parse_list(strings)
-        Bosh::Director::NextRebaseVersion.new(versions).calculate(@version)
+        list = Bosh::Common::Version::ReleaseVersionList.parse(strings)
+        list.rebase(@version)
       end
 
       # Removes release version model, along with all packages and templates.