bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/jobs/helpers/package_deleter.rb

@@ -0,0 +1,33 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class PackageDeleter
+      def initialize(compiled_package_deleter, blob_deleter, logger)
+        @compiled_package_deleter = compiled_package_deleter
+        @blob_deleter = blob_deleter
+        @logger = logger
+      end
+
+      def delete(package, force)
+        errors = []
+        @logger.info("Deleting package #{package.name}/#{package.version}")
+
+        package.compiled_packages.each do |compiled_package|
+          errors += @compiled_package_deleter.delete(compiled_package, {'force' => force})
+        end
+
+        delete_successful = true
+
+        if package.blobstore_id
+          delete_successful = @blob_deleter.delete(package.blobstore_id, errors, force)
+        end
+
+        if delete_successful
+          package.remove_all_release_versions
+          package.destroy
+        end
+
+        errors
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/helpers/release_deleter.rb

@@ -0,0 +1,52 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class ReleaseDeleter
+
+      def initialize(package_deleter, template_deleter, event_log, logger)
+        @package_deleter = package_deleter
+        @template_deleter = template_deleter
+        @event_log = event_log
+        @logger = logger
+      end
+
+      def delete(release, force)
+        errors = []
+
+        stage = @event_log.begin_stage('Deleting packages', release.packages.count)
+        release.packages.each do |package|
+          track_and_log(stage, "#{package.name}/#{package.version}") do
+            errors += @package_deleter.delete(package, force)
+          end
+        end
+
+        stage = @event_log.begin_stage('Deleting jobs', release.templates.count)
+        release.templates.each do |template|
+          track_and_log(stage, "#{template.name}/#{template.version}") do
+            errors += @template_deleter.delete(template, force)
+          end
+        end
+
+        if errors.empty? || force
+          stage = @event_log.begin_stage('Deleting release versions', release.versions.count)
+          release.versions.each do |release_version|
+            track_and_log(stage, "#{release.name}/#{release_version.version}") do
+              release_version.destroy
+            end
+          end
+          release.destroy
+        end
+
+        errors
+      end
+
+      private
+
+      def track_and_log(stage, task, log = true)
+        stage.advance_and_track(task) do |ticker|
+          @logger.info(task) if log
+          yield ticker if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/helpers/release_version_deleter.rb

@@ -0,0 +1,115 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class ReleaseVersionDeleter
+
+      def initialize(release_deleter, package_deleter, template_deleter, logger, event_log)
+        @release_deleter = release_deleter
+        @package_deleter = package_deleter
+        @template_deleter = template_deleter
+        @logger = logger
+        @event_log = event_log
+      end
+
+      def delete(release_version, release, force)
+        @logger.info('Checking for any deployments still using ' +
+            'this particular release version')
+
+        deployments = release_version.deployments
+
+        unless deployments.empty?
+          names = deployments.map { |d| d.name }.join(', ')
+          raise Bosh::Director::ReleaseVersionInUse,
+            "ReleaseVersion `#{release.name}/#{release_version.version}' is still in use by: #{names}"
+        end
+
+        delete_release_version(release_version, force)
+      end
+
+      private
+
+      def delete_release_version(release_version, force)
+        errors = []
+
+        release = release_version.release
+
+        packages_to_keep = []
+        packages_to_delete = []
+        templates_to_keep = []
+        templates_to_delete = []
+
+        # We don't delete packages inside this loop b/c Sequel will also delete
+        # them from packages collection we're iterating on which will lead to
+        # skipping some packages
+        release_version.packages.each do |package|
+          if package.release_versions == [release_version]
+            packages_to_delete << package
+          else
+            packages_to_keep << package
+          end
+        end
+
+        release_version.templates.each do |template|
+          if template.release_versions == [release_version]
+            templates_to_delete << template
+          else
+            templates_to_keep << template
+          end
+        end
+
+
+        stage = @event_log.begin_stage('Deleting packages', packages_to_delete.count)
+        packages_to_delete.each do |package|
+          track_and_log(stage, "#{package.name}/#{package.version}") do
+            @logger.info("Package #{package.name}/#{package.version} " +
+                'is only used by this release version ' +
+                'and will be deleted')
+            errors += @package_deleter.delete(package, force)
+          end
+        end
+
+        packages_to_keep.each do |package|
+          @logger.info("Keeping package #{package.name}/#{package.version} " +
+              'as it is used by other release versions')
+          package.remove_release_version(release_version)
+        end
+
+        stage = @event_log.begin_stage('Deleting jobs', templates_to_delete.count)
+        templates_to_delete.each do |template|
+          track_and_log(stage, "#{template.name}/#{template.version}") do
+            @logger.info("Template #{template.name}/#{template.version} " +
+                'is only used by this release version ' +
+                'and will be deleted')
+            errors += @template_deleter.delete(template, force)
+          end
+        end
+
+        templates_to_keep.each do |template|
+          @logger.info('Keeping job ' +
+              "#{template.name}/#{template.version} as it is used " +
+              'by other release versions')
+          template.remove_release_version(release_version)
+        end
+
+        @logger.info('Remove all deployments in release version')
+        release_version.remove_all_deployments
+
+        if errors.empty? || force
+          release_version.destroy
+        end
+
+        if release.versions.empty?
+          errors += @release_deleter.delete(release, force)
+        end
+
+        errors
+      end
+
+      def track_and_log(stage, task)
+        stage.advance_and_track(task) do
+          @logger.info(task)
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/helpers/releases_to_delete_picker.rb

@@ -0,0 +1,31 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class ReleasesToDeletePicker
+      def initialize(release_manager)
+        @release_manager = release_manager
+      end
+
+      def pick(releases_to_keep)
+        unused_releases = @release_manager
+                            .get_all_releases
+                            .map do |release|
+          release['release_versions'].reject! { |version| version['currently_deployed'] }
+          release
+        end
+
+        unused_releases_to_delete = unused_releases
+                                      .reject{ |release| release['release_versions'].empty? }
+                                      .map do |release|
+          release['release_versions'].pop(releases_to_keep)
+          release
+        end
+
+        unused_releases_to_delete.map do |release|
+          release['release_versions'].map do |version|
+            {'name' => release['name'], 'version' => version['version']}
+          end
+        end.flatten
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/helpers/stemcell_deleter.rb

@@ -0,0 +1,61 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class StemcellDeleter
+      include Bosh::Director::LockHelper
+
+      def initialize(cloud, compiled_package_deleter, logger, event_log)
+        @cloud = cloud
+        @compiled_package_deleter = compiled_package_deleter
+        @logger = logger
+        @event_log = event_log
+      end
+
+      def delete(stemcell, options = {})
+        with_stemcell_lock(stemcell.name, stemcell.version) do
+          @logger.info('Checking for any deployments still using the stemcell')
+          deployments = stemcell.deployments
+          unless deployments.empty?
+            names = deployments.map { |d| d.name }.join(', ')
+            raise Bosh::Director::StemcellInUse,
+              "Stemcell `#{stemcell.name}/#{stemcell.version}' is still in use by: #{names}"
+          end
+
+          begin
+            @cloud.delete_stemcell(stemcell.cid)
+          rescue => e
+            raise unless options['force']
+            @logger.warn(e.backtrace.join("\n"))
+            @logger.info("Force deleting is set, ignoring exception: #{e.message}")
+          end
+
+          @logger.info('Looking for any compiled packages on this stemcell')
+          compiled_packages = Bosh::Director::Models::CompiledPackage.filter(:stemcell_id => stemcell.id)
+
+          stage = @event_log.begin_stage('Deleting compiled packages',
+            compiled_packages.count, [stemcell.name, stemcell.version])
+          @logger.info('Deleting compiled packages ' +
+              "(#{compiled_packages.count}) for `#{stemcell.name}/#{stemcell.version}'")
+
+          compiled_packages.each do |compiled_package|
+            message = "#{compiled_package.name}/#{compiled_package.version}"
+            stage.advance_and_track(message) do
+              @logger.info(message)
+              @compiled_package_deleter.delete(compiled_package, options)
+            end
+          end
+
+          stemcell.destroy
+        end
+      end
+
+      private
+
+      def track_and_log(stage, task, log = true)
+        stage.advance_and_track(task) do |ticker|
+          @logger.info(task) if log
+          yield ticker if block_given?
+        end
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/helpers/stemcells_to_delete_picker.rb

@@ -0,0 +1,30 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class StemcellsToDeletePicker
+      def initialize(stemcell_manager)
+        @stemcell_manager = stemcell_manager
+      end
+
+      def pick(stemcells_to_keep)
+        unused_stemcell_names_and_versions = @stemcell_manager
+                                               .find_all_stemcells
+                                               .select { |stemcell| stemcell['deployments'].empty? }
+
+        unused_stemcells_grouped_by_name = unused_stemcell_names_and_versions.inject({}) do |h, stemcell|
+          h[stemcell['name']] ||= []
+          h[stemcell['name']] << stemcell
+          h
+        end
+
+        stemcells_to_versions_to_delete = unused_stemcells_grouped_by_name.each_pair do |_, versions|
+          versions.sort! do |sc1, sc2|
+            Bosh::Common::Version::StemcellVersion.parse(sc1['version']) <=> Bosh::Common::Version::StemcellVersion.parse(sc2['version'])
+          end
+          versions.pop(stemcells_to_keep)
+        end
+
+        stemcells_to_versions_to_delete.values.flatten
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/helpers/template_deleter.rb

@@ -0,0 +1,20 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class TemplateDeleter
+      def initialize(blob_deleter, logger)
+        @blob_deleter = blob_deleter
+        @logger = logger
+      end
+
+      def delete(template, force)
+        @logger.info("Deleting job: #{template.name}/#{template.version}")
+        errors = []
+        if @blob_deleter.delete(template.blobstore_id, errors, force)
+          template.remove_all_release_versions
+          template.destroy
+        end
+        errors
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/release/release_job.rb

@@ -33,6 +33,8 @@ module Bosh::Director
       template.properties = job_manifest['properties'] if job_manifest['properties']
 
       validate_links(job_manifest)
+      template.provides = job_manifest['provides'] if job_manifest['provides']
+      template.requires = job_manifest['requires'] if job_manifest['requires']
 
       template.save
     end
@@ -146,16 +148,25 @@ module Bosh::Director
     end
 
     def validate_links(job_manifest)
-      if job_manifest['provides']
-        if !job_manifest['provides'].is_a?(Array) || job_manifest['provides'].find { |p| !p.is_a?(String) }
-          raise JobInvalidLinkSpec, "Job `#{@name}' has invalid spec format: 'provides' needs to be an array of strings"
-        end
+      parse_links(job_manifest['provides'], 'provides') if job_manifest['provides']
+      parse_links(job_manifest['requires'], 'requires') if job_manifest['requires']
+    end
+
+    def parse_links(links, desc)
+      if !links.is_a?(Array)
+        raise JobInvalidLinkSpec,
+          "Job '#{@name}' has invalid spec format: '#{desc}' must be an array of strings or hashes with name and type"
       end
 
-      if job_manifest['requires']
-        if !job_manifest['requires'].is_a?(Array) || job_manifest['requires'].find { |p| !p.is_a?(String) }
-          raise JobInvalidLinkSpec, "Job `#{@name}' has invalid spec format: 'requires' needs to be an array of strings"
+      parsed_links = {}
+      links.each do |link_spec|
+        parsed_link = DeploymentPlan::TemplateLink.parse(link_spec)
+        if parsed_links[parsed_link.name]
+          raise JobDuplicateLinkName,
+            "Job '#{@name}' '#{desc}' specifies links with duplicate name '#{parsed_link.name}'"
         end
+
+        parsed_links[parsed_link.name] = true
       end
     end
   end

data/lib/bosh/director/jobs/run_errand.rb

@@ -15,9 +15,9 @@ module Bosh::Director
       @errand_name = errand_name
       @deployment_manager = Api::DeploymentManager.new
       @instance_manager = Api::InstanceManager.new
-      @blobstore = App.instance.blobstores.blobstore
       @keep_alive = keep_alive
-      log_bundles_cleaner = LogBundlesCleaner.new(@blobstore, 60 * 60 * 24 * 10, logger) # 10 days
+      blobstore = App.instance.blobstores.blobstore
+      log_bundles_cleaner = LogBundlesCleaner.new(blobstore, 60 * 60 * 24 * 10, logger) # 10 days
       @logs_fetcher = LogsFetcher.new(event_log, @instance_manager, log_bundles_cleaner, logger)
     end
 
@@ -28,24 +28,38 @@ module Bosh::Director
       with_deployment_lock(deployment_name) do
         cloud_config_model = deployment_model.cloud_config
 
-        planner_factory = DeploymentPlan::PlannerFactory.create(event_log, logger)
-        deployment = planner_factory.planner(deployment_manifest_hash, cloud_config_model, {})
+        deployment = nil
+        job = nil
 
-        job = deployment.job(@errand_name)
-        if job.nil?
-          raise JobNotFound, "Errand `#{@errand_name}' doesn't exist"
-        end
+        event_log.begin_stage('Preparing deployment', 1)
+        event_log.track('Preparing deployment') do
+          planner_factory = DeploymentPlan::PlannerFactory.create(logger)
+          deployment = planner_factory.create_from_manifest(deployment_manifest_hash, cloud_config_model, {})
+          deployment.bind_models
+          job = deployment.job(@errand_name)
 
-        unless job.can_run_as_errand?
-          raise RunErrandError,
-                "Job `#{job.name}' is not an errand. To mark a job as an errand " +
-                  "set its lifecycle to 'errand' in the deployment manifest."
-        end
+          if job.nil?
+            raise JobNotFound, "Errand `#{@errand_name}' doesn't exist"
+          end
+
+          unless job.can_run_as_errand?
+            raise RunErrandError,
+              "Job `#{job.name}' is not an errand. To mark a job as an errand " +
+                "set its lifecycle to 'errand' in the deployment manifest."
+          end
+
+          if job.instances.empty?
+            raise InstanceNotFound, "Instance `#{@deployment_name}/#{@errand_name}/0' doesn't exist"
+          end
+
+          logger.info('Starting to prepare for deployment')
+          job.bind_instances(deployment.ip_provider)
 
-        if job.instances.empty?
-          raise InstanceNotFound, "Instance `#{@deployment_name}/#{@errand_name}/0' doesn't exist"
+          JobRenderer.create.render_job_instances(job.needed_instance_plans)
         end
 
+        deployment.compile_packages
+
         runner = Errand::Runner.new(job, result_file, @instance_manager, event_log, @logs_fetcher)
 
         cancel_blk = lambda {
@@ -71,44 +85,51 @@ module Bosh::Director
     private
 
     def with_updated_instances(deployment, job, &blk)
-      rp_updaters = [ResourcePoolUpdater.new(job.resource_pool)]
-      resource_pools = DeploymentPlan::ResourcePools.new(event_log, rp_updaters)
+      job_manager = Errand::JobManager.new(deployment, job, Config.cloud, event_log, logger)
 
-      job_manager = Errand::JobManager.new(deployment, job, @blobstore, event_log, logger)
+      begin
+        update_instances(job_manager)
+        block_result = blk.call
+      rescue Exception
+        cleanup_instances_and_log_error(job_manager)
+        raise
+      else
+        cleanup_instances_and_raise_error(job_manager)
+        return block_result
+      end
+    end
 
+    def cleanup_instances_and_log_error(job_manager)
       begin
-        update_instances(resource_pools, job_manager)
-        blk.call
-      ensure
-        if @keep_alive
-          logger.info('Skipping instances deletion, keep-alive is set')
-        else
-          logger.info('Deleting instances')
-          delete_instances(resource_pools, job_manager)
-        end
+        cleanup_instances_and_raise_error(job_manager)
+      rescue Exception => e
+        logger.warn("Failed to delete instances: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}")
       end
     end
 
-    def update_instances(resource_pools, job_manager)
-      logger.info('Starting to prepare for deployment')
-      job_manager.prepare
+    def cleanup_instances_and_raise_error(job_manager)
+      if @keep_alive
+        logger.info('Skipping instances deletion, keep-alive is set')
+      else
+        logger.info('Deleting instances')
+        delete_instances(job_manager)
+      end
+    end
 
-      logger.info('Starting to update resource pool')
-      resource_pools.update
+    def update_instances(job_manager)
+      logger.info('Starting to create missing vms')
+      job_manager.create_missing_vms
 
       logger.info('Starting to update job instances')
       job_manager.update_instances
     end
 
-    def delete_instances(resource_pools, job_manager)
+    def delete_instances(job_manager)
       @ignore_cancellation = true
 
       logger.info('Starting to delete job instances')
       job_manager.delete_instances
 
-      logger.info('Starting to refill resource pool')
-      resource_pools.refill
-
       @ignore_cancellation = false
     end
   end