bosh-director 1.2941.0 → 1.2949.0

data/lib/bosh/director/deployment_plan/steps/prepare_step.rb

@@ -0,0 +1,56 @@
+module Bosh::Director
+  module DeploymentPlan
+    module Steps
+      class PrepareStep
+        def initialize(job, assembler)
+          @job = job
+          @assembler = assembler
+        end
+
+        def perform
+          job.begin_stage('Preparing deployment', 9)
+
+          job.track_and_log('Binding deployment') do
+            @assembler.bind_deployment
+          end
+
+          job.track_and_log('Binding releases') do
+            @assembler.bind_releases
+          end
+
+          job.track_and_log('Binding existing deployment') do
+            @assembler.bind_existing_deployment
+          end
+
+          job.track_and_log('Binding resource pools') do
+            @assembler.bind_resource_pools
+          end
+
+          job.track_and_log('Binding stemcells') do
+            @assembler.bind_stemcells
+          end
+
+          job.track_and_log('Binding templates') do
+            @assembler.bind_templates
+          end
+
+          job.track_and_log('Binding properties') do
+            @assembler.bind_properties
+          end
+
+          job.track_and_log('Binding unallocated VMs') do
+            @assembler.bind_unallocated_vms
+          end
+
+          job.track_and_log('Binding instance networks') do
+            @assembler.bind_instance_networks
+          end
+        end
+
+        private
+
+        attr_reader :job
+      end
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/steps/update_step.rb

@@ -0,0 +1,70 @@
+module Bosh::Director
+  module DeploymentPlan
+    module Steps
+      class UpdateStep
+        def initialize(base_job, event_log, resource_pools, assembler, deployment_plan, multi_job_updater)
+          @base_job = base_job
+          @logger = base_job.logger
+          @event_log = event_log
+          @resource_pools = resource_pools
+          @assembler = assembler
+          @deployment_plan = deployment_plan
+          @multi_job_updater = multi_job_updater
+        end
+
+
+        def perform
+          begin
+            @logger.info('Updating deployment')
+            assemble
+            update_jobs
+            @logger.info('Committing updates')
+            @deployment_plan.persist_updates!
+            @logger.info('Finished updating deployment')
+          ensure
+            @deployment_plan.update_stemcell_references!
+          end
+        end
+
+        private
+
+        def assemble
+          @event_log.begin_stage('Preparing DNS', 1)
+          @base_job.track_and_log('Binding DNS') do
+            @assembler.bind_dns
+          end
+
+          @logger.info('Deleting no longer needed VMs')
+          @assembler.delete_unneeded_vms
+
+          @logger.info('Deleting no longer needed instances')
+          @assembler.delete_unneeded_instances
+
+          @logger.info('Updating resource pools')
+          @resource_pools.update
+          @base_job.task_checkpoint
+
+          @logger.info('Binding instance VMs')
+          @assembler.bind_instance_vms
+
+          @event_log.begin_stage('Preparing configuration', 1)
+          @base_job.track_and_log('Binding configuration') do
+            @assembler.bind_configuration
+          end
+        end
+
+        def update_jobs
+          @logger.info('Updating jobs')
+          @multi_job_updater.run(
+            @base_job,
+            @deployment_plan,
+            @deployment_plan.jobs_starting_on_deploy,
+          )
+
+          @logger.info('Refilling resource pools')
+          @resource_pools.refill
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/errand/deployment_preparer.rb

@@ -1,9 +1,6 @@
 module Bosh::Director
   class Errand::DeploymentPreparer
-    # @param [Bosh::Director::DeploymentPlan::Planner] deployment
-    # @param [Bosh::Director::DeploymentPlan::Job] job
-    # @param [Bosh::Director::EventLog::Log] event_log
-    # @param [Bosh::Blobstore::Jobs::BaseJob] base_job
+
     def initialize(deployment, job, event_log, base_job)
       @deployment = deployment
       @job = job
@@ -14,11 +11,11 @@ module Bosh::Director
     def prepare_deployment
       assembler = DeploymentPlan::Assembler.new(@deployment)
 
-      preparer = DeploymentPlan::Preparer.new(@base_job, assembler)
-      preparer.prepare
+      preparer = DeploymentPlan::Steps::PrepareStep.new(@base_job, assembler)
+      preparer.perform
 
-      compiler = PackageCompiler.new(@deployment)
-      compiler.compile
+      compiler = DeploymentPlan::Steps::PackageCompileStep.new(@deployment)
+      compiler.perform
     end
 
     def prepare_job

data/lib/bosh/director/jobs/run_errand.rb

@@ -24,7 +24,7 @@ module Bosh::Director
     def perform
       deployment_model = @deployment_manager.find_by_name(@deployment_name)
       manifest = Psych.load(deployment_model.manifest)
-      deployment = DeploymentPlan::Planner.parse(manifest, {}, event_log, logger)
+      deployment = DeploymentPlan::Planner.parse(manifest, deployment_model.cloud_config, {}, event_log, logger)
 
       job = deployment.job(@errand_name)
       if job.nil?

data/lib/bosh/director/jobs/update_deployment.rb

@@ -3,119 +3,95 @@ module Bosh::Director
     class UpdateDeployment < BaseJob
       include LockHelper
 
-      attr_reader :notifier
-
       @queue = :normal
 
       def self.job_type
         :update_deployment
       end
 
-      # @param [String] manifest_file Path to deployment manifest
-      # @param [Hash] options Deployment options
-      def initialize(manifest_file, cloud_config_id, options = {})
+      def initialize(manifest_file_path, cloud_config_id, options = {})
         @blobstore = App.instance.blobstores.blobstore
+        @manifest_file_path = manifest_file_path
+        @options = options
+        @cloud_config_id = cloud_config_id
+      end
 
-        logger.info('Reading deployment manifest')
-        @manifest_file = manifest_file
-        @manifest = File.read(@manifest_file)
-        logger.debug("Manifest:\n#{@manifest}")
+      def perform
+        with_deployment_lock(deployment_plan) do
+          logger.info('Updating deployment')
+          notifier.send_start_event
+          prepare_step.perform
+          compile_step.perform
+          update_step.perform
+          notifier.send_end_event
+          logger.info('Finished updating deployment')
 
-        @cloud_config = Bosh::Director::Models::CloudConfig.find(id: cloud_config_id)
-        logger.debug("Cloud Config:\n#{@cloud_config.inspect}")
+          "/deployments/#{deployment_plan.name}"
+        end
+      rescue Exception => e
+        notifier.send_error_event e
+        raise e
+      ensure
+        FileUtils.rm_rf(@manifest_file_path)
+      end
 
-        logger.info('Creating deployment plan')
-        logger.info("Deployment plan options: #{options.pretty_inspect}")
+      private
 
-        plan_options = {
-          'recreate' => !!options['recreate'],
-          'job_states' => options['job_states'] || {},
-          'job_rename' => options['job_rename'] || {}
-        }
+      # Job tasks
 
-        manifest_as_hash = Psych.load(@manifest)
-        @deployment_plan = DeploymentPlan::Planner.parse(manifest_as_hash, plan_options, event_log, logger)
-        logger.info('Created deployment plan')
+      def prepare_step
+        DeploymentPlan::Steps::PrepareStep.new(self, assembler)
+      end
 
-        nats_rpc = Config.nats_rpc
-        @notifier = DeploymentPlan::Notifier.new(@deployment_plan, nats_rpc, logger)
+      def compile_step
+        DeploymentPlan::Steps::PackageCompileStep.new(deployment_plan)
+      end
 
-        resource_pools = @deployment_plan.resource_pools
-        @resource_pool_updaters = resource_pools.map do |resource_pool|
+      def update_step
+        resource_pool_updaters = deployment_plan.resource_pools.map do |resource_pool|
           ResourcePoolUpdater.new(resource_pool)
         end
+        resource_pools = DeploymentPlan::ResourcePools.new(event_log, resource_pool_updaters)
+        DeploymentPlan::Steps::UpdateStep.new(self, event_log, resource_pools, assembler, deployment_plan, multi_job_updater)
       end
 
-      def prepare
-        @assembler = DeploymentPlan::Assembler.new(@deployment_plan)
-        preparer = DeploymentPlan::Preparer.new(self, @assembler)
-        preparer.prepare
+      # Job dependencies
 
-        logger.info('Compiling and binding packages')
-        PackageCompiler.new(@deployment_plan).compile
+      def assembler
+        @assembler ||= DeploymentPlan::Assembler.new(deployment_plan)
       end
 
-      def update
-        resource_pools = DeploymentPlan::ResourcePools.new(event_log, @resource_pool_updaters)
-        job_updater_factory = JobUpdaterFactory.new(@blobstore)
-        multi_job_updater = DeploymentPlan::BatchMultiJobUpdater.new(job_updater_factory)
-        updater = DeploymentPlan::Updater.new(self, event_log, resource_pools, @assembler, @deployment_plan, multi_job_updater)
-        updater.update
+      def notifier
+        @notifier ||= DeploymentPlan::Notifier.new(deployment_plan, Config.nats_rpc, logger)
       end
 
-      def update_stemcell_references
-        current_stemcells = Set.new
-        @deployment_plan.resource_pools.each do |resource_pool|
-          current_stemcells << resource_pool.stemcell.model
-        end
-
-        deployment = @deployment_plan.model
-        stemcells = deployment.stemcells
-        stemcells.each do |stemcell|
-          unless current_stemcells.include?(stemcell)
-            stemcell.remove_deployment(deployment)
-          end
+      def deployment_plan
+        @deployment_plan ||= begin
+          logger.info('Reading deployment manifest')
+          manifest_text = File.read(@manifest_file_path)
+          logger.debug("Manifest:\n#{manifest_text}")
+          deployment_manifest = Psych.load(manifest_text)
+
+          plan_options = {
+            'recreate' => !!@options['recreate'],
+            'job_states' => @options['job_states'] || {},
+            'job_rename' => @options['job_rename'] || {}
+          }
+          logger.info('Creating deployment plan')
+          logger.info("Deployment plan options: #{plan_options.pretty_inspect}")
+
+          cloud_config = Bosh::Director::Models::CloudConfig[@cloud_config_id]
+
+          plan = DeploymentPlan::Planner.parse(deployment_manifest, cloud_config, plan_options, event_log, logger)
+          logger.info('Created deployment plan')
+          plan
         end
       end
 
-      def perform
-        with_deployment_lock(@deployment_plan) do
-          logger.info('Preparing deployment')
-          notifier.send_start_event
-          prepare
-          begin
-            deployment = @deployment_plan.model
-            logger.info('Finished preparing deployment')
-            logger.info('Updating deployment')
-            update
-
-            with_release_locks(@deployment_plan) do
-              deployment.db.transaction do
-                deployment.remove_all_release_versions
-                # Now we know that deployment has succeeded and can remove
-                # previous partial deployments release version references
-                # to be able to delete these release versions later.
-                @deployment_plan.releases.each do |release|
-                  deployment.add_release_version(release.model)
-                end
-              end
-            end
-
-            deployment.manifest = @manifest
-            deployment.cloud_config = @cloud_config
-            deployment.save
-            notifier.send_end_event
-            logger.info('Finished updating deployment')
-            "/deployments/#{deployment.name}"
-          ensure
-            update_stemcell_references
-          end
+      def multi_job_updater
+        @multi_job_updater ||= begin
+          DeploymentPlan::BatchMultiJobUpdater.new(JobUpdaterFactory.new(@blobstore))
         end
-      rescue Exception => e
-        notifier.send_error_event e
-        raise e
-      ensure
-        FileUtils.rm_rf(@manifest_file)
       end
     end
   end

data/lib/bosh/director/lock_helper.rb

@@ -1,13 +1,5 @@
 module Bosh::Director
-  # Helper for managing BOSH locks.
   module LockHelper
-
-    # Surround with deployment lock.
-    #
-    # @param [DeploymentPlan|String] deployment plan or name.
-    # @option opts [Number] timeout how long to wait before giving up
-    # @return [void]
-    # @yield [void] block to surround
     def with_deployment_lock(deployment, opts = {})
       if deployment.respond_to?(:name)
         name = deployment.name
@@ -21,13 +13,6 @@ module Bosh::Director
       Lock.new("lock:deployment:#{name}", :timeout => timeout).lock { yield }
     end
 
-    # Surround with stemcell lock.
-    #
-    # @param [String] name stemcell name.
-    # @param [String] version stemcell version.
-    # @option opts [Number] timeout how long to wait before giving up
-    # @return [void]
-    # @yield [void] block to surround
     def with_stemcell_lock(name, version, opts = {})
       timeout = opts[:timeout] || 10
       Config.logger.info("Acquiring stemcell lock on #{name}:#{version}")
@@ -35,31 +20,12 @@ module Bosh::Director
           lock { yield }
     end
 
-    # Surround with release lock.
-    #
-    # @param [String] release name.
-    # @option opts [Number] timeout how long to wait before giving up
-    # @return [void]
-    # @yield [void] block to surround
-    def with_release_lock(release, opts = {})
-      timeout = opts[:timeout] || 10
-      Config.logger.info("Acquiring release lock on #{release}")
-      Lock.new("lock:release:#{release}", :timeout => timeout).lock { yield }
+    def with_release_lock(release_name, opts = {})
+      with_release_locks([release_name], opts) { yield }
     end
 
-    # Surround with deployment releases lock.
-    #
-    # @param [DeploymentPlan] deployment plan.
-    # @option opts [Number] timeout how long to wait before giving up
-    # @return [void]
-    # @yield [void] block to surround
-    def with_release_locks(deployment_plan, opts = {})
+    def with_release_locks(release_names, opts = {})
       timeout = opts[:timeout] || 10
-      release_names = deployment_plan.releases.map do |release|
-        release.name
-      end
-
-      # Sorting to enforce lock order to avoid deadlocks
       locks = release_names.sort.map do |release_name|
         Config.logger.info("Acquiring release lock: #{release_name}")
         Lock.new("lock:release:#{release_name}", :timeout => timeout)
@@ -73,13 +39,6 @@ module Bosh::Director
       end
     end
 
-    # Surround with compile lock.
-    #
-    # @param [String|Number] package_id package id.
-    # @param [String|Number] stemcell_id stemcell id.
-    # @option opts [Number] timeout how long to wait before giving up
-    # @return [void]
-    # @yield [void] block to surround
     def with_compile_lock(package_id, stemcell_id, opts = {})
       timeout = opts[:timeout] || 15 * 60 # 15 minutes
 

data/lib/bosh/director/models/cloud_config.rb

@@ -5,6 +5,14 @@ module Bosh
         def before_create
           self.created_at ||= Time.now
         end
+
+        def manifest=(cloud_config_hash)
+          self.properties = Psych.dump(cloud_config_hash)
+        end
+
+        def manifest
+          Psych.load properties
+        end
       end
     end
   end

data/lib/bosh/director/models/deployment.rb

@@ -9,7 +9,7 @@ module Bosh::Director::Models
     one_to_many  :vms
     one_to_many  :properties, :class => "Bosh::Director::Models::DeploymentProperty"
     one_to_many  :problems, :class => "Bosh::Director::Models::DeploymentProblem"
-    many_to_one   :cloud_config
+    many_to_one  :cloud_config
 
     def validate
       validates_presence :name

data/lib/bosh/director/version.rb

@@ -1,5 +1,5 @@
 module Bosh
   module Director
-    VERSION = '1.2941.0'
+    VERSION = '1.2949.0'
   end
 end