bosh-director 1.2941.0 → 1.2949.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ff8d3b656ced1dac2993e1b47a0bcbe7a7b6e12
-  data.tar.gz: bc540bdd773816f632624ac090e9ea93fb2b014a
+  metadata.gz: f2d58f426d00ab34e503324a356f32419e8ec589
+  data.tar.gz: 50c33061600de723b450e53aee4234e0f84f98b3
 SHA512:
-  metadata.gz: 404b30865cad8f8bb7bbc1e2bc355a6ed24e30194c0c7ca09f953281bd82fa6204cb0621f669de762aeac5cf27a948ad89cf0b74f3a2b81de81afe9d2c948a5b
-  data.tar.gz: 2193105304206a1203c9c31d432e61d7acc33c56cc44bdae9b8eacd33979e96cc868e0212e3b90ace51c127ebe0b7f2cb0afae2e2a9b74aad71e79a4a0710628
+  metadata.gz: 3b4108244748cffa5cd372f29faddc1504f6859904bb95b6eb6ce1a86531f832e6dab33dfa2070e3df9455a3be29716512dce4b4c3b68ea2bdbaa2ded74ac8bd
+  data.tar.gz: aa4dddfd9fd057e4e62f5835e85487bbb4ae18ab4f926acc852cd5eb1cee6ac9f591cfd027067265ef7fef948b3e03e765a482edcf472ea897a87449c841c9c0

data/lib/bosh/director.rb

@@ -81,7 +81,6 @@ require 'bosh/director/job_queue'
 require 'bosh/director/lock'
 require 'bosh/director/nats_rpc'
 require 'bosh/director/network_reservation'
-require 'bosh/director/package_compiler'
 require 'bosh/director/problem_scanner/scanner'
 require 'bosh/director/problem_resolver'
 require 'bosh/director/resource_pool_updater'

data/lib/bosh/director/api/controllers/deployments_controller.rb

@@ -294,7 +294,7 @@ module Bosh::Director
         deployment = @deployment_manager.find_by_name(params[:deployment_name])
 
         manifest = Psych.load(deployment.manifest)
-        deployment_plan = DeploymentPlan::Planner.parse(manifest, {}, Config.event_log, Config.logger)
+        deployment_plan = DeploymentPlan::Planner.parse(manifest, deployment.cloud_config, {}, Config.event_log, Config.logger)
 
         errands = deployment_plan.jobs.select(&:can_run_as_errand?)
 

data/lib/bosh/director/config.rb

@@ -10,7 +10,7 @@ module Bosh::Director
     class << self
       include DnsHelper
 
-      CONFIG_OPTIONS = [
+      attr_accessor(
         :base_dir,
         :cloud_options,
         :db,
@@ -33,17 +33,13 @@ module Bosh::Director
         :enable_snapshots,
         :max_vm_create_tries,
         :nats_uri,
-      ]
-
-      CONFIG_OPTIONS.each do |option|
-        attr_accessor option
-      end
+      )
 
       attr_reader :db_config, :redis_logger_level
 
       def clear
-        CONFIG_OPTIONS.each do |option|
-          self.instance_variable_set("@#{option}".to_sym, nil)
+        self.instance_variables.each do |ivar|
+          self.instance_variable_set(ivar, nil)
         end
 
         Thread.list.each do |thr|

data/lib/bosh/director/deployment_plan.rb

@@ -10,11 +10,9 @@ require 'bosh/director/deployment_plan/job'
 require 'bosh/director/deployment_plan/network'
 require 'bosh/director/deployment_plan/network_subnet'
 require 'bosh/director/deployment_plan/compiled_package'
-require 'bosh/director/deployment_plan/preparer'
 require 'bosh/director/deployment_plan/resource_pools'
 require 'bosh/director/deployment_plan/instance_vm_binder'
 require 'bosh/director/deployment_plan/multi_job_updater'
-require 'bosh/director/deployment_plan/updater'
 require 'bosh/director/deployment_plan/release_version'
 require 'bosh/director/deployment_plan/resource_pool'
 require 'bosh/director/deployment_plan/stemcell'
@@ -26,3 +24,7 @@ require 'bosh/director/deployment_plan/vip_network'
 require 'bosh/director/deployment_plan/planner'
 require 'bosh/director/deployment_plan/dns_binder'
 require 'bosh/director/deployment_plan/notifier'
+require 'bosh/director/deployment_plan/steps/prepare_step'
+require 'bosh/director/deployment_plan/steps/update_step'
+require 'bosh/director/deployment_plan/steps/package_compile_step'
+

data/lib/bosh/director/deployment_plan/assembler.rb

@@ -23,8 +23,9 @@ module Bosh::Director
     # Binds release DB record(s) to a plan
     # @return [void]
     def bind_releases
-      with_release_locks(@deployment_plan) do
-        @deployment_plan.releases.each do |release|
+      releases = @deployment_plan.releases
+      with_release_locks(releases.map(&:name)) do
+        releases.each do |release|
           release.bind_model
         end
       end

data/lib/bosh/director/deployment_plan/deployment_spec_parser.rb

@@ -13,13 +13,17 @@ module Bosh::Director
 
       # @param [Hash] manifest Raw deployment manifest
       # @return [DeploymentPlan::Planner] Deployment as build from deployment_spec
-      def parse(manifest, options = {})
-        @manifest = manifest
+      def parse(manifest, cloud_config, options = {})
+        @deployment_manifest = manifest
+        if cloud_config.nil?
+          @cloud_manifest = cloud_manifest_from_deployment_manifest @deployment_manifest 
+        else
+          @cloud_manifest = cloud_config.manifest
+        end
 
-        @job_states = safe_property(options, 'job_states',
-          :class => Hash, :default => {})
+        @job_states = safe_property(options, 'job_states', :class => Hash, :default => {})
 
-        @deployment = Planner.new(parse_name, options)
+        @deployment = Planner.new(parse_name, manifest, cloud_config, options)
 
         parse_properties
         parse_releases
@@ -35,27 +39,36 @@ module Bosh::Director
 
       private
 
+      CLOUD_MANIFEST_KEYS = ['resource_pools','compilation','disk_pools','networks']
+      def cloud_manifest_from_deployment_manifest(deployment_manifest)
+        cloud_manifest = {}
+        CLOUD_MANIFEST_KEYS.each do |key|
+          cloud_manifest[key] = deployment_manifest[key] if deployment_manifest.has_key? key
+        end
+        cloud_manifest
+      end
+
       def parse_name
-        safe_property(@manifest, 'name', :class => String)
+        safe_property(@deployment_manifest, 'name', :class => String)
       end
 
       def parse_properties
-        @deployment.properties = safe_property(@manifest, 'properties',
+        @deployment.properties = safe_property(@deployment_manifest, 'properties',
           :class => Hash, :default => {})
       end
 
       def parse_releases
         release_specs = []
 
-        if @manifest.has_key?('release')
-          if @manifest.has_key?('releases')
+        if @deployment_manifest.has_key?('release')
+          if @deployment_manifest.has_key?('releases')
             raise DeploymentAmbiguousReleaseSpec,
               "Deployment manifest contains both 'release' and 'releases' " +
                 'sections, please use one of the two.'
           end
-          release_specs << @manifest['release']
+          release_specs << @deployment_manifest['release']
         else
-          safe_property(@manifest, 'releases', :class => Array).each do |release|
+          safe_property(@deployment_manifest, 'releases', :class => Array).each do |release|
             release_specs << release
           end
         end
@@ -66,7 +79,7 @@ module Bosh::Director
       end
 
       def parse_networks
-        networks = safe_property(@manifest, 'networks', :class => Array)
+        networks = safe_property(@cloud_manifest, 'networks', :class => Array)
         networks.each do |network_spec|
           type = safe_property(network_spec, 'type', :class => String,
             :default => 'manual')
@@ -92,17 +105,17 @@ module Bosh::Director
       end
 
       def parse_compilation
-        compilation_spec = safe_property(@manifest, 'compilation', :class => Hash)
+        compilation_spec = safe_property(@cloud_manifest, 'compilation', :class => Hash)
         @deployment.compilation = CompilationConfig.new(@deployment, compilation_spec)
       end
 
       def parse_update
-        update_spec = safe_property(@manifest, 'update', :class => Hash)
+        update_spec = safe_property(@deployment_manifest, 'update', :class => Hash)
         @deployment.update = UpdateConfig.new(update_spec)
       end
 
       def parse_resource_pools
-        resource_pools = safe_property(@manifest, 'resource_pools', :class => Array)
+        resource_pools = safe_property(@cloud_manifest, 'resource_pools', :class => Array)
         resource_pools.each do |rp_spec|
           @deployment.add_resource_pool(ResourcePool.new(@deployment, rp_spec, @logger))
         end
@@ -112,7 +125,7 @@ module Bosh::Director
       end
 
       def parse_disk_pools
-        disk_pools = safe_property(@manifest, 'disk_pools', :class => Array, :optional => true)
+        disk_pools = safe_property(@cloud_manifest, 'disk_pools', :class => Array, :optional => true)
         return if disk_pools.nil?
         disk_pools.each do |dp_spec|
           @deployment.add_disk_pool(DiskPool.parse(dp_spec))
@@ -120,7 +133,7 @@ module Bosh::Director
       end
 
       def parse_jobs
-        jobs = safe_property(@manifest, 'jobs', :class => Array, :default => [])
+        jobs = safe_property(@deployment_manifest, 'jobs', :class => Array, :default => [])
         jobs.each do |job_spec|
           state_overrides = @job_states[job_spec['name']]
           if state_overrides

data/lib/bosh/director/deployment_plan/planner.rb

@@ -6,6 +6,7 @@ module Bosh::Director
   # from the deployment manifest and the running environment.
   module DeploymentPlan
     class Planner
+      include LockHelper
       include DnsHelper
       include ValidationHelper
 
@@ -53,16 +54,18 @@ module Bosh::Director
       # @param [Logger]
       #   logger Log for director logging
       # @return [Bosh::Director::DeploymentPlan::Planner]
-      def self.parse(manifest, options, event_log, logger)
+      def self.parse(manifest, cloud_config, options, event_log, logger)
         parser = DeploymentSpecParser.new(event_log, logger)
-        parser.parse(manifest, options)
+        parser.parse(manifest, cloud_config, options)
       end
 
-      def initialize(name, options = {})
+      def initialize(name, manifest_text, cloud_config, options = {})
         raise ArgumentError, 'name must not be nil' unless name
         @name = name
-        @model = nil
+        @manifest_text = manifest_text
+        @cloud_config = cloud_config
 
+        @model = nil
         @properties = {}
         @releases = {}
         @networks = {}
@@ -260,6 +263,29 @@ module Bosh::Director
       def rename_in_progress?
         @job_rename['old_name'] && @job_rename['new_name']
       end
+
+      def persist_updates!
+        #prior updates may have had release versions that we no longer use.
+        #remove the references to these stale releases.
+        stale_release_versions = (model.release_versions - releases.map(&:model))
+        stale_release_names = stale_release_versions.map {|version_model| version_model.release.name}
+        with_release_locks(stale_release_names) do
+          stale_release_versions.each do |release_version|
+            model.remove_release_version(release_version)
+          end
+        end
+
+        model.manifest = Psych.dump(@manifest_text)
+        model.cloud_config = @cloud_config
+        model.save
+      end
+
+      def update_stemcell_references!
+        current_stemcell_models = resource_pools.map { |pool| pool.stemcell.model }
+        model.stemcells.each do |deployment_stemcell|
+          deployment_stemcell.remove_deployment(model) unless current_stemcell_models.include?(deployment_stemcell)
+        end
+      end
     end
   end
 end

data/lib/bosh/director/deployment_plan/steps/package_compile_step.rb

@@ -0,0 +1,340 @@
+require 'bosh/director/compile_task_generator'
+
+module Bosh::Director
+  module DeploymentPlan
+    module Steps
+      class PackageCompileStep
+        include LockHelper
+
+        attr_reader :compilations_performed
+
+        # @param [DeploymentPlan] deployment_plan Deployment plan
+        def initialize(deployment_plan)
+          @deployment_plan = deployment_plan
+
+          @cloud = Config.cloud
+          @event_log = Config.event_log
+          @logger = Config.logger
+          @director_job = Config.current_job
+
+          @tasks_mutex = Mutex.new
+          @network_mutex = Mutex.new
+          @counter_mutex = Mutex.new
+
+          compilation_config = @deployment_plan.compilation
+
+          @network = compilation_config.network
+          @compilation_resources = compilation_config.cloud_properties
+          @compilation_env = compilation_config.env
+
+          @vm_reuser = VmReuser.new
+
+          @compile_task_generator = CompileTaskGenerator.new(@logger, @event_log)
+
+          @compile_tasks = {}
+          @ready_tasks = []
+          @compilations_performed = 0
+        end
+
+        def perform
+          @logger.info('Generating a list of compile tasks')
+          prepare_tasks
+
+          @compile_tasks.each_value do |task|
+            if task.ready_to_compile?
+              @logger.info("Package `#{task.package.desc}' is ready to be compiled for stemcell `#{task.stemcell.desc}'")
+              @ready_tasks << task
+            end
+          end
+
+          if @ready_tasks.empty?
+            @logger.info('All packages are already compiled')
+          else
+            compile_packages
+            director_job_checkpoint
+          end
+        end
+
+        def compile_tasks_count
+          @compile_tasks.size
+        end
+
+        def ready_tasks_count
+          @tasks_mutex.synchronize { @ready_tasks.size }
+        end
+
+
+
+        def compile_package(task)
+          package = task.package
+          stemcell = task.stemcell
+
+          with_compile_lock(package.id, stemcell.id) do
+            # Check if the package was compiled in a parallel deployment
+            compiled_package = task.find_compiled_package(@logger, @event_log)
+            if compiled_package.nil?
+              build = Models::CompiledPackage.generate_build_number(package, stemcell)
+              task_result = nil
+
+              prepare_vm(stemcell) do |vm_data|
+                vm_metadata_updater.update(vm_data.vm, :compiling => package.name)
+                agent_task =
+                  vm_data.agent.compile_package(package.blobstore_id,
+                                                package.sha1, package.name,
+                                                "#{package.version}.#{build}",
+                                                task.dependency_spec)
+                task_result = agent_task['result']
+              end
+
+              compiled_package = Models::CompiledPackage.create do |p|
+                p.package = package
+                p.stemcell = stemcell
+                p.sha1 = task_result['sha1']
+                p.build = build
+                p.blobstore_id = task_result['blobstore_id']
+                p.dependency_key = task.dependency_key
+              end
+
+              if Config.use_compiled_package_cache?
+                if BlobUtil.exists_in_global_cache?(package, task.cache_key)
+                  @logger.info('Already exists in global package cache, skipping upload')
+                else
+                  @logger.info('Uploading to global package cache')
+                  BlobUtil.save_to_global_cache(compiled_package, task.cache_key)
+                end
+              else
+                @logger.info('Global blobstore not configured, skipping upload')
+              end
+
+              @counter_mutex.synchronize { @compilations_performed += 1 }
+            end
+
+            task.use_compiled_package(compiled_package)
+          end
+        end
+
+        # This method will create a VM for each stemcell in the stemcells array
+        # passed in.  The VMs are yielded and their destruction is ensured.
+        # @param [Models::Stemcell] stemcell The stemcells that need to have
+        #     compilation VMs created.
+        # @yield [VmData] Yields a VmData object that contains all the data for the
+        #     VM that should be used for compilation.  This may be a reused VM or a
+        #     freshly created VM.
+        def prepare_vm(stemcell)
+          # If we're reusing VMs, try to just return an already-created VM.
+          if @deployment_plan.compilation.reuse_compilation_vms
+            vm_data = @vm_reuser.get_vm(stemcell)
+            if vm_data
+              @logger.info("Reusing compilation VM `#{vm_data.vm.cid}' for stemcell `#{stemcell.desc}'")
+              begin
+                yield vm_data
+              ensure
+                vm_data.release
+              end
+              return
+            end
+            # This shouldn't happen. If it does there's a bug.
+            if @vm_reuser.get_num_vms(stemcell) >=
+              @deployment_plan.compilation.workers
+              raise PackageCompilationNotEnoughWorkersForReuse,
+                    'There should never be more VMs for a stemcell than the number of workers in reuse_compilation_vms mode'
+            end
+          end
+
+          @logger.info("Creating compilation VM for stemcell `#{stemcell.desc}'")
+
+          reservation = reserve_network
+
+          network_settings = {
+            @network.name => @network.network_settings(reservation)
+          }
+
+          vm = VmCreator.create(@deployment_plan.model, stemcell,
+                                @compilation_resources, network_settings,
+                                nil, @compilation_env)
+          vm_data = @vm_reuser.add_vm(reservation, vm, stemcell, network_settings)
+
+          @logger.info("Configuring compilation VM: #{vm.cid}")
+
+          begin
+            agent = AgentClient.with_defaults(vm.agent_id)
+            agent.wait_until_ready
+
+            configure_vm(vm, agent, network_settings)
+            vm_data.agent = agent
+            yield vm_data
+          rescue RpcTimeout => e
+            # if we time out waiting for the agent, we should clean up the the VM
+            # as it will leave us in an unrecoverable state otherwise
+            @vm_reuser.remove_vm(vm_data)
+            tear_down_vm(vm_data)
+            raise e
+          ensure
+            vm_data.release
+            unless @deployment_plan.compilation.reuse_compilation_vms
+              tear_down_vm(vm_data)
+            end
+          end
+        end
+
+        private
+
+        def prepare_tasks
+          @event_log.begin_stage('Preparing package compilation', 1)
+
+          @event_log.track('Finding packages to compile') do
+            @deployment_plan.jobs.each do |job|
+              stemcell = job.resource_pool.stemcell
+
+              template_descs = job.templates.map do |t|
+                # we purposefully did NOT inline those because
+                # when instance_double blows up,
+                # it's obscure which double is at fault
+                release_name = t.release.name
+                template_name = t.name
+                "`#{release_name}/#{template_name}'"
+              end
+              @logger.info("Job templates #{template_descs.join(', ')} need to run on stemcell `#{stemcell.model.desc}'")
+
+              job.templates.each do |template|
+                template.package_models.each do |package|
+                  @compile_task_generator.generate!(@compile_tasks, job, template, package, stemcell.model)
+                end
+              end
+            end
+          end
+        end
+
+        def tear_down_vm(vm_data)
+          vm = vm_data.vm
+          if vm.exists?
+            reservation = vm_data.reservation
+            @logger.info("Deleting compilation VM: #{vm.cid}")
+            @cloud.delete_vm(vm.cid)
+            vm.destroy
+            release_network(reservation)
+          end
+        end
+
+        def reserve_network
+          reservation = NetworkReservation.new_dynamic
+
+          @network_mutex.synchronize do
+            @network.reserve(reservation)
+          end
+
+          unless reservation.reserved?
+            raise PackageCompilationNetworkNotReserved,
+                  "Could not reserve network for package compilation: #{reservation.error}"
+          end
+
+          reservation
+        end
+
+        def release_network(reservation)
+          @network_mutex.synchronize do
+            @network.release(reservation)
+          end
+        end
+
+        def compile_packages
+          @event_log.begin_stage('Compiling packages', compilation_count)
+          number_of_workers = @deployment_plan.compilation.workers
+
+          begin
+            ThreadPool.new(:max_threads => number_of_workers).wrap do |pool|
+              loop do
+                # process as many tasks without waiting
+                loop do
+                  break if director_job_cancelled?
+                  task = @tasks_mutex.synchronize { @ready_tasks.pop }
+                  break if task.nil?
+
+                  pool.process { process_task(task) }
+                end
+
+                break if !pool.working? && (director_job_cancelled? || @ready_tasks.empty?)
+                sleep(0.1)
+              end
+            end
+          ensure
+            # Delete all of the VMs if we were reusing compilation VMs. This can't
+            # happen until everything was done compiling.
+            if @deployment_plan.compilation.reuse_compilation_vms
+              # Using a new ThreadPool instead of reusing the previous one,
+              # as if there's a failed compilation, the thread pool will stop
+              # processing any new thread.
+              ThreadPool.new(:max_threads => number_of_workers).wrap do |pool|
+                @vm_reuser.each do |vm_data|
+                  pool.process { tear_down_vm(vm_data) }
+                end
+              end
+            end
+          end
+        end
+
+        def enqueue_unblocked_tasks(task)
+          @tasks_mutex.synchronize do
+            @logger.info("Unblocking dependents of `#{task.package.desc}` for `#{task.stemcell.desc}`")
+            task.dependent_tasks.each do |dep_task|
+              if dep_task.ready_to_compile?
+                @logger.info("Package `#{dep_task.package.desc}' now ready to be compiled for `#{dep_task.stemcell.desc}'")
+                @ready_tasks << dep_task
+              end
+            end
+          end
+        end
+
+        def process_task(task)
+          package_desc = task.package.desc
+          stemcell_desc = task.stemcell.desc
+          task_desc = "package `#{package_desc}' for stemcell `#{stemcell_desc}'"
+
+          with_thread_name("compile_package(#{package_desc}, #{stemcell_desc})") do
+            if director_job_cancelled?
+              @logger.info("Cancelled compiling #{task_desc}")
+            else
+              @event_log.track(package_desc) do
+                @logger.info("Compiling #{task_desc}")
+                compile_package(task)
+                @logger.info("Finished compiling #{task_desc}")
+                enqueue_unblocked_tasks(task)
+              end
+            end
+          end
+        end
+
+        def director_job_cancelled?
+          @director_job && @director_job.task_cancelled?
+        end
+
+        def director_job_checkpoint
+          @director_job.task_checkpoint if @director_job
+        end
+
+        def configure_vm(vm, agent, network_settings)
+          state = {
+            'deployment' => @deployment_plan.name,
+            'resource_pool' => 'package_compiler',
+            'networks' => network_settings
+          }
+
+          vm.update(:apply_spec => state)
+          agent.apply(state)
+        end
+
+        def compilation_count
+          counter = 0
+          @compile_tasks.each_value do |task|
+            counter += 1 unless task.compiled?
+          end
+          counter
+        end
+
+        def vm_metadata_updater
+          @vm_metadata_updater ||= VmMetadataUpdater.build
+        end
+      end
+    end
+  end
+end