bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/jobs/delete_stemcell.rb

@@ -1,101 +1,35 @@
-# Copyright (c) 2009-2012 VMware, Inc.
-
 module Bosh::Director
   module Jobs
     class DeleteStemcell < BaseJob
-      include LockHelper
-
       @queue = :normal
 
       def self.job_type
         :delete_stemcell
       end
 
-      # @param [String] name Stemcell name
-      # @param [String] version Stemcell version
       def initialize(name, version, options = {})
         @name = name
         @version = version
         @options = options
         @cloud = Config.cloud
-        @blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
         @stemcell_manager = Api::StemcellManager.new
+
+        blobstore = options.fetch(:blobstore) { App.instance.blobstores.blobstore }
+        blob_deleter = Helpers::BlobDeleter.new(blobstore, logger)
+        compiled_package_deleter = Helpers::CompiledPackageDeleter.new(blob_deleter, logger)
+        @stemcell_deleter = Helpers::StemcellDeleter.new(@cloud, compiled_package_deleter, logger, event_log)
       end
 
       def perform
         logger.info("Processing delete stemcell")
-        with_stemcell_lock(@name, @version) do
-          logger.info("Looking up stemcell: #{desc}")
-          @stemcell = @stemcell_manager.find_by_name_and_version(@name, @version)
-          logger.info("Found: #{@stemcell.pretty_inspect}")
-
-          validate_deletion
-          delete_from_cloud
-          cleanup_compiled_packages
-          delete_stemcell_metadata
-        end
-
-        "/stemcells/#{@name}/#{@version}"
-      end
-
-      def validate_deletion
-        logger.info("Checking for any deployments still using the stemcell")
-        deployments = @stemcell.deployments
-        unless deployments.empty?
-          names = deployments.map { |d| d.name }.join(", ")
-          raise StemcellInUse,
-                "Stemcell `#{desc}' is still in use by: #{names}"
-        end
-      end
-
-      def delete_from_cloud
-        event_log.begin_stage("Deleting stemcell from cloud", 1)
 
-        event_log.track("Delete stemcell") do
-          @cloud.delete_stemcell(@stemcell.cid)
-        end
-      rescue => e
-        raise unless force?
-        logger.warn(e.backtrace.join("\n"))
-        logger.info("Force deleting is set, ignoring exception: #{e.message}")
-      end
-
-      def cleanup_compiled_packages
-        logger.info("Looking for any compiled packages on this stemcell")
-        compiled_packages =
-            Models::CompiledPackage.filter(:stemcell_id => @stemcell.id)
-
-        event_log.begin_stage("Deleting compiled packages",
-                              compiled_packages.count, [@name, @version])
-        logger.info("Deleting compiled packages " +
-                        "(#{compiled_packages.count}) for `#{desc}'")
-
-        compiled_packages.each do |compiled_package|
-          next unless compiled_package
-
-          package = compiled_package.package
-          track_and_log("#{package.name}/#{package.version}") do
-            logger.info("Deleting compiled package: " +
-                            "#{package.name}/#{package.version}")
-            @blobstore.delete(compiled_package.blobstore_id)
-            compiled_package.destroy
-          end
-        end
-      end
+        logger.info("Looking up stemcell: #{@name}/#{@version}")
+        stemcell = @stemcell_manager.find_by_name_and_version(@name, @version)
+        logger.info("Found: #{stemcell.pretty_inspect}")
 
-      def delete_stemcell_metadata
-        event_log.begin_stage("Deleting stemcell metadata", 1)
-        event_log.track("Deleting stemcell metadata") do
-          @stemcell.destroy
-        end
-      end
+        @stemcell_deleter.delete(stemcell, @options)
 
-      def desc
-        "#@name/#@version"
-      end
-
-      def force?
-        @options["force"]
+        "/stemcells/#{@name}/#{@version}"
       end
     end
   end

data/lib/bosh/director/jobs/export_release.rb

@@ -27,44 +27,44 @@ module Bosh::Director
       def perform
         logger.info("Exporting release: #{@release_name}/#{@release_version} for #{@stemcell_os}/#{@stemcell_version}")
 
-        stemcell_manager = Bosh::Director::Api::StemcellManager.new
-        @stemcell = stemcell_manager.find_by_os_and_version(@stemcell_os, @stemcell_version)
-
-        logger.info "Will compile with stemcell: #{@stemcell.desc}"
+        deployment_plan_stemcell = Bosh::Director::DeploymentPlan::Stemcell.new({
+            "os" => @stemcell_os,
+            "version" => @stemcell_version
+          })
 
         deployment_manager = Bosh::Director::Api::DeploymentManager.new
-        @targeted_deployment = deployment_manager.find_by_name(@deployment_name)
-        @deployment_manifest = Psych.load(@targeted_deployment.manifest)
+        targeted_deployment = deployment_manager.find_by_name(@deployment_name)
 
         release_manager = Bosh::Director::Api::ReleaseManager.new
         release = release_manager.find_by_name(@release_name)
-        @release_version_model = release_manager.find_version(release, @release_version)
+        release_version_model = release_manager.find_version(release, @release_version)
 
-        unless deployment_manifest_has_release?
+        unless deployment_manifest_has_release?(targeted_deployment.manifest)
           raise ReleaseNotMatchingManifest, "Release version `#{@release_name}/#{@release_version}' not found in deployment `#{@deployment_name}' manifest"
         end
 
-        validate_release_packages
+        planner_factory = DeploymentPlan::PlannerFactory.create(logger)
+        planner = planner_factory.create_from_model(targeted_deployment)
+        deployment_plan_stemcell.bind_model(planner)
+
+        stemcell_model = deployment_plan_stemcell.model
+        logger.info "Will compile with stemcell: #{stemcell_model.desc}"
+
+        release = planner.release(@release_name)
+
+        export_release_job = create_job_with_all_the_templates_so_everything_compiles(release_version_model, release, planner, deployment_plan_stemcell)
+        planner.add_job(export_release_job)
+        planner.bind_models
 
         lock_timeout = 15 * 60 # 15 minutes
 
         with_deployment_lock(@deployment_name, :timeout => lock_timeout) do
           with_release_lock(@release_name, :timeout => lock_timeout) do
-            with_stemcell_lock(@stemcell.name, @stemcell.version, :timeout => lock_timeout) do
-
-              planner = create_planner(stemcell_manager)
-              package_compile_step = DeploymentPlan::Steps::PackageCompileStep.new(
-                  planner,
-                  Config.cloud, # CPI
-                  Config.logger,
-                  Config.event_log,
-                  self
-              )
-              package_compile_step.perform
-
-              tarball_state = create_tarball
-              result_file.write(tarball_state.to_json + "\n")
+            with_stemcell_lock(stemcell_model.name, stemcell_model.version, :timeout => lock_timeout) do
+              planner.compile_packages
 
+              tarball_state = create_tarball(release_version_model, stemcell_model)
+              result_file.write(tarball_state.to_json + "\n")
             end
           end
         end
@@ -73,71 +73,26 @@ module Bosh::Director
 
       private
 
-      def deployment_manifest_has_release?
-        @deployment_manifest["releases"].each do |release|
-          if (release["name"] == @release_name) && (release["version"].to_s == @release_version.to_s)
+      def deployment_manifest_has_release?(manifest)
+        deployment_manifest = Psych.load(manifest)
+        deployment_manifest['releases'].each do |release|
+          if (release['name'] == @release_name) && (release['version'].to_s == @release_version.to_s)
             return true
           end
         end
         false
       end
 
-      def create_planner(stemcell_manager)
-        cloud_config_model = @targeted_deployment.cloud_config
-
-        planner_factory = DeploymentPlan::PlannerFactory.create(Config.event_log, Config.logger)
-        planner = planner_factory.planner_without_vm_binding(
-            @deployment_manifest,
-            cloud_config_model,
-            {}
-        )
-        network_name = planner.networks.first.name
-
-        fake_resource_pool_manifest = {
-            "name" => "just_for_compiling",
-            "network" => network_name,
-            "stemcell" => { "name" => @stemcell.name, "version" => @stemcell.version }
-        }
-
-        resource_pool = DeploymentPlan::ResourcePool.new(planner, fake_resource_pool_manifest, Config.logger)
-        planner.add_resource_pool(resource_pool)
-        planner.reset_jobs
-
-        fake_job = create_fake_job(planner, fake_resource_pool_manifest, network_name)
-        planner.add_job(fake_job)
-
-        assembler = DeploymentPlan::Assembler.new(
-            planner,
-            stemcell_manager,
-            Config.cloud,
-            nil, # blobstore not used for this assembler purposes
-            @logger,
-            @event_log
-        )
-        @logger.info('Created deployment plan')
-
-        track_and_log('Binding existing deployment') do
-          assembler.bind_existing_deployment
-        end
-
-        track_and_log('Binding resource pools') do
-          assembler.bind_resource_pools
-        end
-
-        planner
-      end
-
-      def create_tarball
-
+      def create_tarball(release_version_model, stemcell_model)
         blobstore_client = Bosh::Director::App.instance.blobstores.blobstore
 
-        compiled_packages_group = CompiledPackageGroup.new(@release_version_model, @stemcell)
-        templates = @release_version_model.templates.map
+        compiled_packages_group = CompiledPackageGroup.new(release_version_model, stemcell_model)
+        templates = release_version_model.templates.map
 
         compiled_release_downloader = CompiledReleaseDownloader.new(compiled_packages_group, templates, blobstore_client)
         download_dir = compiled_release_downloader.download
 
-        manifest = CompiledReleaseManifest.new(compiled_packages_group, templates, @stemcell)
+        manifest = CompiledReleaseManifest.new(compiled_packages_group, templates, stemcell_model)
         manifest.write(File.join(download_dir, 'release.MF'))
 
         output_path = File.join(download_dir, "compiled_release_#{Time.now.to_f}.tar.gz")
@@ -149,60 +104,25 @@ module Bosh::Director
         oid = blobstore_client.create(tarball_file)
 
         {
-            :blobstore_id => oid,
-            :sha1 => Digest::SHA1.file(output_path).hexdigest,
+          :blobstore_id => oid,
+          :sha1 => Digest::SHA1.file(output_path).hexdigest,
         }
       ensure
         compiled_release_downloader.cleanup unless compiled_release_downloader.nil?
       end
 
-      def create_fake_job(planner, fake_resource_pool_manifest, network_name)
-        fake_job_spec_for_compiling = {
-            "name" => "dummy-job-for-compilation",
-            "release" => @release_name,
-            "instances" => 1,
-            "resource_pool" => fake_resource_pool_manifest['name'],
-            "templates" => @release_version_model.templates.map do |template|
-              { "name" => template.name, "release" => @release_name }
-            end,
-            "networks" => [ "name" => network_name ],
-        }
+      def create_job_with_all_the_templates_so_everything_compiles(release_version_model, release, planner, deployment_plan_stemcell)
+        job = DeploymentPlan::Job.new(logger)
 
-        fake_job = DeploymentPlan::Job.parse(planner, fake_job_spec_for_compiling, Config.event_log, Config.logger)
-        @release_version_model.packages.each { |package| fake_job.packages[package.name] = package }
-        fake_job.resource_pool.stemcell.bind_model
-        fake_job.release.bind_model
-        fake_job.templates.each { |template| template.bind_models }
-        fake_job
-      end
-
-      def validate_release_packages
-        faults = Set.new
-        @release_version_model.packages.each do |package|
-          packages_list = @release_version_model.transitive_dependencies(package)
-          packages_list << package
-
-          packages_list.each { |needed_package|
-            if needed_package.sha1.nil? || needed_package.blobstore_id.nil?
-              compiled_packages_list = Bosh::Director::Models::CompiledPackage[:package_id => needed_package.id, :stemcell_id => @stemcell.id]
-              if compiled_packages_list.nil?
-                faults << needed_package
-              end
-            end
-          }
+        job.name = 'dummy-job-for-compilation'
+        job.stemcell = deployment_plan_stemcell
+        job.all_properties = planner.properties
+        release_version_model.templates.map do |template|
+          job.templates << release.get_or_create_template(template.name)
         end
 
-        unless faults.empty?
-          sorted_faults = faults.to_a.sort_by { |p| p.name }
-          msg = "Can't export release `#{@release_name}/#{@release_version}'. It references packages without" +
-              " source code that are not compiled against `#{@stemcell.desc}':\n"
-          sorted_faults.each do |non_compiled_package|
-            msg += " - #{non_compiled_package.name}/#{non_compiled_package.version}\n"
-          end
-          raise PackageMissingSourceCode, msg
-        end
+        job
       end
-
     end
   end
 end

data/lib/bosh/director/jobs/fetch_logs.rb

@@ -22,14 +22,8 @@ module Bosh::Director
 
       def perform
         instance = @instance_manager.find_instance(@instance_id)
-        desc = "#{instance.job}/#{instance.index}"
 
         deployment = instance.deployment
-        if deployment.nil?
-          raise InstanceDeploymentMissing,
-                "`#{desc}' doesn't belong to any deployment"
-        end
-
         with_deployment_lock(deployment) do
           @logs_fetcher.fetch(instance, @log_type, @filters)
         end

data/lib/bosh/director/jobs/helpers.rb

@@ -0,0 +1,10 @@
+require 'bosh/director/jobs/helpers/releases_to_delete_picker'
+require 'bosh/director/jobs/helpers/stemcells_to_delete_picker'
+require 'bosh/director/jobs/helpers/stemcell_deleter'
+require 'bosh/director/jobs/helpers/compiled_package_deleter'
+require 'bosh/director/jobs/helpers/package_deleter'
+require 'bosh/director/jobs/helpers/blob_deleter'
+require 'bosh/director/jobs/helpers/template_deleter'
+require 'bosh/director/jobs/helpers/release_version_deleter'
+require 'bosh/director/jobs/helpers/release_deleter'
+require 'bosh/director/jobs/helpers/name_version_release_deleter'

data/lib/bosh/director/jobs/helpers/blob_deleter.rb

@@ -0,0 +1,24 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class BlobDeleter
+
+      def initialize(blobstore, logger)
+        @blobstore = blobstore
+        @logger = logger
+      end
+
+      def delete(blobstore_id, errors, force)
+        begin
+          @blobstore.delete(blobstore_id)
+          return true
+        rescue Exception => e
+          @logger.warn("Could not delete blob with id `#{blobstore_id}`from blobstore: #{e}\n " + e.backtrace.join("\n"))
+          errors << e
+        end
+
+        force
+      end
+    end
+  end
+end
+

data/lib/bosh/director/jobs/helpers/compiled_package_deleter.rb

@@ -0,0 +1,24 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class CompiledPackageDeleter
+      def initialize(blob_deleter, logger)
+        @blob_deleter = blob_deleter
+        @logger = logger
+      end
+
+      def delete(compiled_package, options = {})
+        package = compiled_package.package
+        stemcell = compiled_package.stemcell
+        @logger.info('Deleting compiled package: ' +
+            "#{package.name}/#{package.version}" +
+            "for #{stemcell.name}/#{stemcell.version}")
+
+        errors = []
+        if @blob_deleter.delete(compiled_package.blobstore_id, errors,  options['force'])
+          compiled_package.destroy
+        end
+        errors
+      end
+    end
+  end
+end

data/lib/bosh/director/jobs/helpers/name_version_release_deleter.rb

@@ -0,0 +1,48 @@
+module Bosh::Director::Jobs
+  module Helpers
+    class NameVersionReleaseDeleter
+      def initialize(release_deleter, release_manager, release_version_deleter, logger)
+        @release_deleter = release_deleter
+        @release_manager = release_manager
+        @release_version_deleter = release_version_deleter
+        @logger = logger
+      end
+
+      def find_and_delete_release(name, version, force)
+        @logger.info("Looking up release: #{name}")
+        release = @release_manager.find_by_name(name)
+        @logger.info("Found release: #{release.name}")
+
+        if version
+          delete_release_version(release, version, force)
+        else
+          delete_entire_release(release, force)
+        end
+      end
+
+      private
+
+      def delete_entire_release(release, force)
+        @logger.info('Checking for any deployments still using the release')
+        deployments = release.versions.map { |version|
+          version.deployments
+        }.flatten.uniq
+
+        unless deployments.empty?
+          names = deployments.map { |d| d.name }.join(', ')
+          raise Bosh::Director::ReleaseInUse,
+            "Release `#{release.name}' is still in use by: #{names}"
+        end
+        @release_deleter.delete(release, force)
+      end
+
+      def delete_release_version(release, version, force)
+        @logger.info("Looking up release version `#{release.name}/#{version}'")
+        release_version = @release_manager.find_version(release, version)
+        # found version may be different than the requested version, due to version formatting
+        @logger.info("Found release version: `#{release.name}/#{release_version.version}'")
+        @release_version_deleter.delete(release_version, release, force)
+      end
+    end
+  end
+end