bosh-director 1.3048.0 → 1.3050.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99e55b602be5167be9b84464a78041c5062b3a6b
-  data.tar.gz: 50a15495b6c79bba2264981fa4fe07158d6456ed
+  metadata.gz: 21b3597db4028e01b8fe540f0a551c60f2b0e06f
+  data.tar.gz: 5069ad495cd05d71565b6b31cf6a9f68219a8e6e
 SHA512:
-  metadata.gz: 16088e13da325245d18e3f0c45983f1fd288459acb7411475c15a4a7a141b6ae53a62a774ee4b93ccd768e6a987f544e8d16f0956e79884762e484777b53a6a7
-  data.tar.gz: 2ec85bdee426f93026360a03aac14b8b86dca556ce2f60ce6e55f29d3861ab5dbb500c401a4c4e42c193edd366597b9ff2a0f999bbff86efe272d2ccbe1ca0ce
+  metadata.gz: 560901cbd41cc4b0b9a56322797301f7a6e5b1c892a78e763c60fd2df739be8984a076eaf677daeaaaf6f7a27bc4c1100af98e84cd09bc1ea2363f7c28d99218
+  data.tar.gz: 2e1da9546ec0ad35a8d7e33c60638498e882959d8d9626861adbd290b9402a8f820ec379e772f5c1e89eb9a7a797f8fd663a97903a666947e1cc5f10c70ec2e8

data/lib/bosh/director/api/controllers/packages_controller.rb

@@ -16,7 +16,25 @@ module Bosh::Director
           fingerprint_list << package['fingerprint'] if package['fingerprint']
         end
 
-        matching_packages = Models::Package.where(fingerprint: fingerprint_list).all
+        matching_packages = Models::Package.where(fingerprint: fingerprint_list, ~:sha1 => nil, ~:blobstore_id => nil).all
+
+        json_encode(matching_packages.map(&:fingerprint).compact.uniq)
+      end
+
+      post '/matches_compiled', :consumes => :yaml do
+        manifest = Psych.load(request.body)
+
+        unless manifest.is_a?(Hash) && manifest['compiled_packages'].is_a?(Array)
+          raise BadManifest, "Manifest doesn't have a usable packages section"
+        end
+
+        fingerprint_list = []
+
+        manifest['compiled_packages'].each do |package|
+          fingerprint_list << package['fingerprint'] if package['fingerprint']
+        end
+
+        matching_packages = Models::Package.join(Models::CompiledPackage, :package_id=>:id).where(fingerprint: fingerprint_list).all
 
         json_encode(matching_packages.map(&:fingerprint).compact.uniq)
       end

data/lib/bosh/director/api/controllers/releases_controller.rb

@@ -7,7 +7,6 @@ module Bosh::Director
         payload = json_decode(request.body)
         options = {
           rebase:         params['rebase'] == 'true',
-          skip_if_exists: params['skip_if_exists'] == 'true',
         }
         task = @release_manager.create_release_from_url(current_user, payload['location'], options)
         redirect "/tasks/#{task.id}"
@@ -129,7 +128,7 @@ module Bosh::Director
               'blobstore_id' => package.blobstore_id,
               'sha1' => package.sha1,
               'fingerprint' => package.fingerprint.to_s,
-              'compiled_packages' => package.compiled_packages.sort_by { |cp| cp.stemcell }.map do |compiled|
+              'compiled_packages' => package.compiled_packages.sort_by { |cp| [cp.stemcell.name, cp.stemcell.version] }.map do |compiled|
                 {
                     'stemcell' => "#{compiled.stemcell.name}/#{compiled.stemcell.version}",
                     'sha1' => compiled.sha1,

data/lib/bosh/director/api/route_configuration.rb

@@ -12,10 +12,6 @@ module Bosh
           controllers = {}
           controllers['/backups'] = Bosh::Director::Api::Controllers::BackupsController.new(@config)
           controllers['/cloud_configs'] = Bosh::Director::Api::Controllers::CloudConfigsController.new(@config)
-          controllers['/compiled_package_groups'] = Bosh::Director::Api::Controllers::CompiledPackagesController.new(
-            @config,
-            Bosh::Director::Api::CompiledPackageGroupManager.new
-          )
           controllers['/deployments'] = Bosh::Director::Api::Controllers::DeploymentsController.new(@config)
           controllers['/info'] = Bosh::Director::Api::Controllers::InfoController.new(@config)
           controllers['/locks'] = Bosh::Director::Api::Controllers::LocksController.new(@config)

data/lib/bosh/director/api.rb

@@ -19,7 +19,6 @@ require 'bosh/director/api/release_manager'
 require 'bosh/director/api/resource_manager'
 require 'bosh/director/api/snapshot_manager'
 require 'bosh/director/api/stemcell_manager'
-require 'bosh/director/api/compiled_package_group_manager'
 require 'bosh/director/api/task_manager'
 require 'bosh/director/api/user/config_user_manager'
 require 'bosh/director/api/user/database_user_manager'

data/lib/bosh/director/deployment_plan/planner_factory.rb

@@ -56,32 +56,6 @@ module Bosh
           assemble_without_vm_binding(attrs, deployment_manifest, cloud_manifest, deployment_model, cloud_config, options)
         end
 
-        def self.validate_packages(planner, options)
-          release_manager = Bosh::Director::Api::ReleaseManager.new
-          planner.jobs.each { |job|
-            job.templates.each{ |template|
-              release_model = release_manager.find_by_name(template.release.name)
-              template.package_models.each{ |package|
-
-                release_version_model = release_manager.find_version(release_model, template.release.version)
-                packages_list = release_version_model.transitive_dependencies(package)
-                packages_list << package
-
-                packages_list.each { |needed_package|
-                  if needed_package.sha1.nil? || needed_package.blobstore_id.nil?
-                    compiled_packages_list = Bosh::Director::Models::CompiledPackage[:package_id => needed_package.id, :stemcell_id => job.resource_pool.stemcell.model.id]
-                    if compiled_packages_list.nil?
-                      msg = "Can't #{options[:context]} `#{release_version_model.release.name}/#{release_version_model.version}': it is not " +
-                          "compiled for `#{job.resource_pool.stemcell.model.desc}' and no source package is available"
-                      raise PackageMissingSourceCode, msg
-                    end
-                  end
-                }
-              }
-            }
-          }
-        end
-
         private
 
         def deployment_name(manifest_hash)
@@ -121,7 +95,7 @@ module Bosh
 
           run_prepare_step(assembler)
 
-          self.class.validate_packages(planner, {:context => 'deploy'})
+          validate_packages(planner)
 
           DeploymentPlan::Steps::PackageCompileStep.new(
             planner,
@@ -138,6 +112,46 @@ module Bosh
           planner
         end
 
+        def validate_packages(planner)
+          faults = {}
+          release_manager = Bosh::Director::Api::ReleaseManager.new
+          planner.jobs.each { |job|
+            job.templates.each{ |template|
+              release_model = release_manager.find_by_name(template.release.name)
+              template.package_models.each{ |package|
+
+                release_version_model = release_manager.find_version(release_model, template.release.version)
+                packages_list = release_version_model.transitive_dependencies(package)
+                packages_list << package
+
+                release_desc = "#{release_version_model.release.name}/#{release_version_model.version}"
+
+                packages_list.each { |needed_package|
+                  if needed_package.sha1.nil? || needed_package.blobstore_id.nil?
+                    compiled_packages_list = Bosh::Director::Models::CompiledPackage[:package_id => needed_package.id, :stemcell_id => job.resource_pool.stemcell.model.id]
+                    if compiled_packages_list.nil?
+                      (faults[release_desc] ||= []) << {:package => needed_package, :stemcell => job.resource_pool.stemcell.model}
+                    end
+                  end
+                }
+              }
+            }
+          }
+          handle_faults(faults) unless faults.empty?
+        end
+
+        def handle_faults(faults)
+          msg = "\n"
+          faults.each { |release_desc, packages_and_stemcells_list|
+            msg += "\nCan't deploy release `#{release_desc}'. It references packages (see below) without source code and are not compiled against intended stemcells:\n"
+            sorted_packages_and_stemcells = packages_and_stemcells_list.sort_by { |p| p[:package].name }
+            sorted_packages_and_stemcells.each { |item|
+              msg += " - `#{item[:package].name}/#{item[:package].version}' against `#{item[:stemcell].desc}'\n"
+            }
+          }
+          raise PackageMissingSourceCode, msg
+        end
+
         def run_prepare_step(assembler)
           @event_log.begin_stage('Preparing deployment', 9)
           @logger.info('Preparing deployment')

data/lib/bosh/director/errors.rb

@@ -74,6 +74,7 @@ module Bosh::Director
   ReleaseDeleteFailed = err(30009)
   ReleaseVersionInvalid = err(30010)
   ReleaseNotMatchingManifest = err(30011)
+  ReleaseInvalidPackage = err(30012)
 
   ValidationInvalidType = err(40000)
   ValidationMissingField = err(40001)

data/lib/bosh/director/jobs/delete_release.rb

@@ -63,7 +63,7 @@ module Bosh::Director
           package.remove_release_version(release_version)
         end
 
-        event_log.begin_stage("Deleting templates", templates_to_delete.count)
+        event_log.begin_stage("Deleting jobs", templates_to_delete.count)
         templates_to_delete.each do |template|
           track_and_log("#{template.name}/#{template.version}") do
             logger.info("Template #{template.name}/#{template.version} " +
@@ -74,7 +74,7 @@ module Bosh::Director
         end
 
         templates_to_keep.each do |template|
-          logger.info("Keeping template " +
+          logger.info("Keeping job " +
                       "#{template.name}/#{template.version} as it is used " +
                       "by other release versions")
           template.remove_release_version(release_version)
@@ -100,7 +100,7 @@ module Bosh::Director
           end
         end
 
-        event_log.begin_stage("Deleting templates", release.templates.count)
+        event_log.begin_stage("Deleting jobs", release.templates.count)
         release.templates.each do |template|
           track_and_log("#{template.name}/#{template.version}") do
             delete_template(template)
@@ -145,7 +145,7 @@ module Bosh::Director
       end
 
       def delete_template(template)
-        logger.info("Deleting template: #{template.name}/#{template.version}")
+        logger.info("Deleting job: #{template.name}/#{template.version}")
 
         if delete_blobstore_id(template.blobstore_id)
           template.remove_all_release_versions

data/lib/bosh/director/jobs/export_release.rb

@@ -2,6 +2,7 @@ require 'securerandom'
 require 'common/version/release_version'
 require 'bosh/director/compiled_release_downloader'
 require 'bosh/director/compiled_release_manifest'
+require 'bosh/director/compiled_package_group'
 
 module Bosh::Director
   module Jobs
@@ -40,17 +41,18 @@ module Bosh::Director
         @release_version_model = release_manager.find_version(release, @release_version)
 
         unless deployment_manifest_has_release?
-          raise ReleaseNotMatchingManifest, "Release #{@release_name}/#{@release_version} not found in deployment #{@deployment_name} manifest"
+          raise ReleaseNotMatchingManifest, "Release version `#{@release_name}/#{@release_version}' not found in deployment `#{@deployment_name}' manifest"
         end
 
+        validate_release_packages
+
         lock_timeout = 15 * 60 # 15 minutes
 
         with_deployment_lock(@deployment_name, :timeout => lock_timeout) do
           with_release_lock(@release_name, :timeout => lock_timeout) do
             with_stemcell_lock(@stemcell.name, @stemcell.version, :timeout => lock_timeout) do
 
-              planner = create_planner
-              DeploymentPlan::PlannerFactory.validate_packages(planner, {:context => 'export'})
+              planner = create_planner(stemcell_manager)
               package_compile_step = DeploymentPlan::Steps::PackageCompileStep.new(
                   planner,
                   Config.cloud, # CPI
@@ -69,6 +71,8 @@ module Bosh::Director
         "Exported release: #{@release_name}/#{@release_version} for #{@stemcell_os}/#{@stemcell_version}"
       end
 
+      private
+
       def deployment_manifest_has_release?
         @deployment_manifest["releases"].each do |release|
           if (release["name"] == @release_name) && (release["version"].to_s == @release_version.to_s)
@@ -78,7 +82,7 @@ module Bosh::Director
         false
       end
 
-      def create_planner
+      def create_planner(stemcell_manager)
         cloud_config_model = @targeted_deployment.cloud_config
 
         planner_factory = DeploymentPlan::PlannerFactory.create(Config.event_log, Config.logger)
@@ -102,6 +106,24 @@ module Bosh::Director
         fake_job = create_fake_job(planner, fake_resource_pool_manifest, network_name)
         planner.add_job(fake_job)
 
+        assembler = DeploymentPlan::Assembler.new(
+            planner,
+            stemcell_manager,
+            Config.cloud,
+            nil, # blobstore not used for this assembler purposes
+            @logger,
+            @event_log
+        )
+        @logger.info('Created deployment plan')
+
+        track_and_log('Binding existing deployment') do
+          assembler.bind_existing_deployment
+        end
+
+        track_and_log('Binding resource pools') do
+          assembler.bind_resource_pools
+        end
+
         planner
       end
 
@@ -154,7 +176,33 @@ module Bosh::Director
         fake_job
       end
 
+      def validate_release_packages
+        faults = Set.new
+        @release_version_model.packages.each do |package|
+          packages_list = @release_version_model.transitive_dependencies(package)
+          packages_list << package
+
+          packages_list.each { |needed_package|
+            if needed_package.sha1.nil? || needed_package.blobstore_id.nil?
+              compiled_packages_list = Bosh::Director::Models::CompiledPackage[:package_id => needed_package.id, :stemcell_id => @stemcell.id]
+              if compiled_packages_list.nil?
+                faults << needed_package
+              end
+            end
+          }
+        end
+
+        unless faults.empty?
+          sorted_faults = faults.to_a.sort_by { |p| p.name }
+          msg = "Can't export release `#{@release_name}/#{@release_version}'. It references packages without" +
+              " source code that are not compiled against `#{@stemcell.desc}':\n"
+          sorted_faults.each do |non_compiled_package|
+            msg += " - #{non_compiled_package.name}/#{non_compiled_package.version}\n"
+          end
+          raise PackageMissingSourceCode, msg
+        end
+      end
+
     end
   end
 end
-