bosh-director 1.3215.4.0 → 1.3232.0

data/lib/bosh/director/deployment_plan/planner.rb

@@ -121,7 +121,8 @@ module Bosh::Director
         vm_deleter = VmDeleter.new(cloud, @logger)
         disk_manager = DiskManager.new(cloud, @logger)
         job_renderer = JobRenderer.create
-        vm_creator = Bosh::Director::VmCreator.new(cloud, @logger, vm_deleter, disk_manager, job_renderer)
+        arp_flusher = ArpFlusher.new
+        vm_creator = Bosh::Director::VmCreator.new(cloud, @logger, vm_deleter, disk_manager, job_renderer, arp_flusher)
         dns_manager = DnsManagerProvider.create
         instance_deleter = Bosh::Director::InstanceDeleter.new(ip_provider, dns_manager, disk_manager)
         compilation_instance_pool = CompilationInstancePool.new(
@@ -136,7 +137,6 @@ module Bosh::Director
           compilation,
           compilation_instance_pool,
           @logger,
-          Config.event_log,
           nil
         )
         package_compile_step.perform
@@ -179,7 +179,7 @@ module Bosh::Director
       def add_release(release)
         if @releases.has_key?(release.name)
           raise DeploymentDuplicateReleaseName,
-            "Duplicate release name `#{release.name}'"
+            "Duplicate release name '#{release.name}'"
         end
         @releases[release.name] = release
       end

data/lib/bosh/director/deployment_plan/planner_factory.rb

@@ -115,11 +115,11 @@ module Bosh
 
               if template.link_infos.has_key?(current_job.name) && template.link_infos[current_job.name].has_key?('provides')
                 template.link_infos[current_job.name]['provides'].each do |link_name, provided_link|
-                  if provided_link['properties']
+                  if provided_link['link_properties_exported']
                     ## Get default values for this job
                     default_properties = get_default_properties(deployment, template)
 
-                    provided_link['mapped_properties'] = process_link_properties(scoped_properties, default_properties, provided_link['properties'], errors)
+                    provided_link['mapped_properties'] = process_link_properties(scoped_properties, default_properties, provided_link['link_properties_exported'], errors)
                   end
                 end
               end
@@ -168,52 +168,50 @@ module Bosh
 
         def process_link_properties(scoped_properties, default_properties, link_property_list, errors)
           mapped_properties = {}
-
-          link_property_list.each do |link_property| #list of properties
-            previous_property_in_loop = {}
-            current_property_in_loop = scoped_properties
-            mapped_properties_in_loop = mapped_properties
-
-            use_defaults = false
-            property_path = link_property.split(".")
-            property_path.each do |key|
-              if !current_property_in_loop || !current_property_in_loop.has_key?(key)
-                use_defaults = true
-              else
-                current_property_in_loop = current_property_in_loop[key]
-              end
-
-              if !mapped_properties_in_loop.has_key?(key)
-                mapped_properties_in_loop[key] = {}
-              end
-              previous_property_in_loop = mapped_properties_in_loop
-              mapped_properties_in_loop = mapped_properties_in_loop[key]
-            end
-
-            if use_defaults
-              if default_properties.has_key?('properties') && default_properties['properties'].has_key?(link_property)
-                if default_properties['properties'][link_property].has_key?('default')
-                  previous_property_in_loop[property_path.last()] = default_properties['properties'][link_property]['default']
+            link_property_list.each do |link_property|
+              property_path = link_property.split(".")
+              result = find_property(property_path, scoped_properties)
+              if !result['found']
+                if default_properties.has_key?('properties') && default_properties['properties'].has_key?(link_property)
+                  if default_properties['properties'][link_property].has_key?('default')
+                    mapped_properties = update_mapped_properties(mapped_properties, property_path, default_properties['properties'][link_property]['default'])
+                  else
+                    mapped_properties = update_mapped_properties(mapped_properties, property_path, nil)
+                  end
                 else
-                  e = Exception.new("Link property #{link_property} in template #{default_properties['template_name']} has no default value or value supplied by the deployment manifest")
+                  e = Exception.new("Link property #{link_property} in template #{default_properties['template_name']} is not defined in release spec")
                   errors.push(e)
                 end
               else
-                e = Exception.new("Link property #{link_property} in template #{default_properties['template_name']} is not defined in release spec")
-                errors.push(e)
+                mapped_properties = update_mapped_properties(mapped_properties, property_path, result['value'])
               end
+            end
+          return mapped_properties
+        end
+
+        def find_property(property_path, scoped_properties)
+          current_node = scoped_properties
+          property_path.each do |key|
+            if !current_node || !current_node.has_key?(key)
+              return {'found'=> false, 'value' => nil}
             else
-              previous_property_in_loop[property_path.last()] = current_property_in_loop
+              current_node = current_node[key]
             end
+          end
+          return {'found'=> true,'value'=> current_node}
+        end
 
-            # if use_defaults && !default_properties.has_key?("properties") && !default_properties['properties'][link_property].has_key?('default')
-            #   e = Exception.new("Property #{link_property} in template #{default_properties['template_name']} has no default value or value supplied by the deployment manifest")
-            #   errors.push(e)
-            # elsif use_defaults
-            #   previous_property_in_loop[property_path.last()] = default_properties['properties'][link_property]['default']
-            # else
-            #   previous_property_in_loop[property_path.last()] = current_property_in_loop
-            # end
+        def update_mapped_properties(mapped_properties, property_path, value)
+          current_node = mapped_properties
+          property_path.each_with_index do |key, index|
+            if index == property_path.size - 1
+              current_node[key] = value
+            else
+              if !current_node.has_key?(key)
+                current_node[key] = {}
+              end
+              current_node = current_node[key]
+            end
           end
           return mapped_properties
         end

data/lib/bosh/director/deployment_plan/release_version.rb

@@ -35,11 +35,11 @@ module Bosh::Director
 
         release = @manager.find_by_name(@name)
         @model = @manager.find_version(release, @version)
-        @logger.debug("Found release `#{@name}/#{@version}'")
+        @logger.debug("Found release '#{@name}/#{@version}'")
 
         unless @deployment_model.release_versions.include?(@model)
-          @logger.debug("Binding release `#{@name}/#{@version}' " +
-                        "to deployment `#{@deployment_model.name}'")
+          @logger.debug("Binding release '#{@name}/#{@version}' " +
+                        "to deployment '#{@deployment_model.name}'")
           @deployment_model.add_release_version(@model)
         end
       end
@@ -58,9 +58,9 @@ module Bosh::Director
         # have been parsed, so we can assume @templates contains
         # the list of templates that need to be bound
         @templates.each_value do |template|
-          @logger.debug("Binding template `#{template.name}'")
+          @logger.debug("Binding template '#{template.name}'")
           template.bind_models
-          @logger.debug("Bound template `#{template.name}'")
+          @logger.debug("Bound template '#{template.name}'")
         end
       end
 

data/lib/bosh/director/deployment_plan/runtime_manifest_parser.rb

@@ -21,7 +21,7 @@ module Bosh::Director
         if runtime_manifest['release']
           if runtime_manifest['releases']
             raise RuntimeAmbiguousReleaseSpec,
-                  "Runtime manifest contains both `release' and `releases' " +
+                  "Runtime manifest contains both 'release' and 'releases' " +
                       'sections, please use one of the two.'
           end
           @release_specs << runtime_manifest['release']
@@ -34,16 +34,16 @@ module Bosh::Director
         @release_specs.each do |release_spec|
           if release_spec['version'] == 'latest'
             raise RuntimeInvalidReleaseVersion,
-                  "Runtime manifest contains the release `#{release_spec['name']}' with version as `latest'. " +
+                  "Runtime manifest contains the release '#{release_spec['name']}' with version as 'latest'. " +
                       "Please specify the actual version string."
           end
 
           if @deployment
             deployment_release = @deployment.release(release_spec["name"])
             if deployment_release
-              if deployment_release.version != release_spec["version"]
-                raise RuntimeInvalidDeploymentRelease, "Runtime manifest specifies release `#{release_spec["name"]}' with version as `#{release_spec["version"]}'. " +
-                      "This conflicts with version `#{deployment_release.version}' specified in the deployment manifest."
+              if deployment_release.version != release_spec["version"].to_s
+                raise RuntimeInvalidDeploymentRelease, "Runtime manifest specifies release '#{release_spec["name"]}' with version as '#{release_spec["version"]}'. " +
+                      "This conflicts with version '#{deployment_release.version}' specified in the deployment manifest."
               else
                 next
               end
@@ -64,10 +64,10 @@ module Bosh::Director
 
           addon_jobs = safe_property(addon_spec, 'jobs', :class => Array, :default => [])
 
-          addon_jobs.each do |job|
-            if !@release_specs.find { |release_spec| release_spec['name'] == job['release'] }
+          addon_jobs.each do |addon_job|
+            if !@release_specs.find { |release_spec| release_spec['name'] == addon_job['release'] }
               raise RuntimeReleaseNotListedInReleases,
-                    "Runtime manifest specifies job `#{job['name']}' which is defined in `#{job['release']}', but `#{job['release']}' is not listed in the releases section."
+                    "Runtime manifest specifies job '#{addon_job['name']}' which is defined in '#{addon_job['release']}', but '#{addon_job['release']}' is not listed in the releases section."
             end
 
             if @deployment
@@ -75,38 +75,42 @@ module Bosh::Director
               deployment_release_ids = Models::Release.where(:name => valid_release_versions).map {|r| r.id}
               deployment_jobs = @deployment.jobs
 
-              templates_from_model = Models::Template.where(:name => job['name'], :release_id => deployment_release_ids)
+              templates_from_model = Models::Template.where(:name => addon_job['name'], :release_id => deployment_release_ids)
               if templates_from_model == nil
-                raise "Job '#{job['name']}' not found in Template table"
+                raise "Job '#{addon_job['name']}' not found in Template table"
               end
 
-              release = @deployment.release(job['release'])
+              release = @deployment.release(addon_job['release'])
               release.bind_model
 
-              template = DeploymentPlan::Template.new(release, job['name'])
+              template = DeploymentPlan::Template.new(release, addon_job['name'])
 
               deployment_jobs.each do |j|
                 templates_from_model.each do |template_from_model|
                   if template_from_model.consumes != nil
                     template_from_model.consumes.each do |consumes|
-                      template.add_link_info(j.name, 'consumes', consumes["name"], consumes)
+                      template.add_link_from_release(j.name, 'consumes', consumes["name"], consumes)
                     end
                   end
                   if template_from_model.provides != nil
                     template_from_model.provides.each do |provides|
-                      template.add_link_info(j.name, 'provides', provides["name"], provides)
+                      template.add_link_from_release(j.name, 'provides', provides["name"], provides)
                     end
                   end
                 end
 
-                provides_links = safe_property(job, 'provides', class: Hash, optional: true)
+                provides_links = safe_property(addon_job, 'provides', class: Hash, optional: true)
                 provides_links.to_a.each do |link_name, source|
-                  template.add_link_info(j.name, "provides", link_name, source)
+                  template.add_link_from_manifest(j.name, "provides", link_name, source)
                 end
 
-                consumes_links = safe_property(job, 'consumes', class: Hash, optional: true)
+                consumes_links = safe_property(addon_job, 'consumes', class: Hash, optional: true)
                 consumes_links.to_a.each do |link_name, source|
-                  template.add_link_info(j.name, 'consumes', link_name, source)
+                  template.add_link_from_manifest(j.name, 'consumes', link_name, source)
+                end
+
+                if addon_job.has_key?('properties')
+                  template.add_template_scoped_properties(addon_job['properties'], j.name)
                 end
               end
 

data/lib/bosh/director/deployment_plan/steps/package_compile_step.rb

@@ -9,8 +9,8 @@ module Bosh::Director
 
         attr_reader :compilations_performed
 
-        def initialize(jobs_to_compile, compilation_config, compilation_instance_pool, logger, event_log, director_job)
-          @event_log = event_log
+        def initialize(jobs_to_compile, compilation_config, compilation_instance_pool, logger, director_job)
+          @event_log_stage = nil
           @logger = logger
           @director_job = director_job
 
@@ -18,9 +18,6 @@ module Bosh::Director
           @counter_mutex = Mutex.new
 
           @compilation_instance_pool = compilation_instance_pool
-
-          @compile_task_generator = CompileTaskGenerator.new(@logger, @event_log)
-
           @compile_tasks = {}
           @ready_tasks = []
           @compilations_performed = 0
@@ -34,7 +31,7 @@ module Bosh::Director
 
           @compile_tasks.each_value do |task|
             if task.ready_to_compile?
-              @logger.info("Package `#{task.package.desc}' is ready to be compiled for stemcell `#{task.stemcell.desc}'")
+              @logger.info("Package '#{task.package.desc}' is ready to be compiled for stemcell '#{task.stemcell.desc}'")
               @ready_tasks << task
             end
           end
@@ -61,7 +58,7 @@ module Bosh::Director
 
           with_compile_lock(package.id, stemcell.id) do
             # Check if the package was compiled in a parallel deployment
-            compiled_package = task.find_compiled_package(@logger, @event_log)
+            compiled_package = task.find_compiled_package(@logger, @event_log_stage)
             if compiled_package.nil?
               build = Models::CompiledPackage.generate_build_number(package, stemcell.model.operating_system, stemcell.model.version)
               task_result = nil
@@ -124,9 +121,10 @@ module Bosh::Director
         private
 
         def prepare_tasks
-          @event_log.begin_stage('Preparing package compilation', 1)
+          @event_log_stage = Config.event_log.begin_stage('Preparing package compilation', 1)
+          @compile_task_generator = CompileTaskGenerator.new(@logger, @event_log_stage)
 
-          @event_log.track('Finding packages to compile') do
+          @event_log_stage.advance_and_track('Finding packages to compile') do
             @jobs_to_compile.each do |job|
               stemcell = job.stemcell
 
@@ -136,9 +134,9 @@ module Bosh::Director
                 # it's obscure which double is at fault
                 release_name = t.release.name
                 template_name = t.name
-                "`#{release_name}/#{template_name}'"
+                "'#{release_name}/#{template_name}'"
               end
-              @logger.info("Job templates #{template_descs.join(', ')} need to run on stemcell `#{stemcell.desc}'")
+              @logger.info("Job templates #{template_descs.join(', ')} need to run on stemcell '#{stemcell.desc}'")
 
               job.templates.each do |template|
                 template.package_models.each do |package|
@@ -150,7 +148,7 @@ module Bosh::Director
         end
 
         def compile_packages
-          @event_log.begin_stage('Compiling packages', compilation_count)
+          @event_log_stage = Config.event_log.begin_stage('Compiling packages', compilation_count)
 
           begin
             ThreadPool.new(:max_threads => @compilation_config.workers).wrap do |pool|
@@ -182,10 +180,10 @@ module Bosh::Director
 
         def enqueue_unblocked_tasks(task)
           @tasks_mutex.synchronize do
-            @logger.info("Unblocking dependents of `#{task.package.desc}` for `#{task.stemcell.desc}`")
+            @logger.info("Unblocking dependents of '#{task.package.desc}' for '#{task.stemcell.desc}'")
             task.dependent_tasks.each do |dep_task|
               if dep_task.ready_to_compile?
-                @logger.info("Package `#{dep_task.package.desc}' now ready to be compiled for `#{dep_task.stemcell.desc}'")
+                @logger.info("Package '#{dep_task.package.desc}' now ready to be compiled for '#{dep_task.stemcell.desc}'")
                 @ready_tasks << dep_task
               end
             end
@@ -195,13 +193,13 @@ module Bosh::Director
         def process_task(task)
           package_desc = task.package.desc
           stemcell_desc = task.stemcell.desc
-          task_desc = "package `#{package_desc}' for stemcell `#{stemcell_desc}'"
+          task_desc = "package '#{package_desc}' for stemcell '#{stemcell_desc}'"
 
           with_thread_name("compile_package(#{package_desc}, #{stemcell_desc})") do
             if director_job_cancelled?
               @logger.info("Cancelled compiling #{task_desc}")
             else
-              @event_log.track(package_desc) do
+              @event_log_stage.advance_and_track(package_desc) do
                 @logger.info("Compiling #{task_desc}")
                 compile_package(task)
                 @logger.info("Finished compiling #{task_desc}")

data/lib/bosh/director/deployment_plan/steps/update_step.rb

@@ -2,17 +2,17 @@ module Bosh::Director
   module DeploymentPlan
     module Steps
       class UpdateStep
-        def initialize(base_job, event_log, deployment_plan, multi_job_updater, cloud)
+        def initialize(base_job, deployment_plan, multi_job_updater, cloud)
           @base_job = base_job
           @logger = base_job.logger
-          @event_log = event_log
           @cloud = cloud
           @deployment_plan = deployment_plan
           @multi_job_updater = multi_job_updater
           @vm_deleter = Bosh::Director::VmDeleter.new(@cloud, @logger)
           @disk_manager = DiskManager.new(@cloud, @logger)
           job_renderer = JobRenderer.create
-          @vm_creator = Bosh::Director::VmCreator.new(@cloud, @logger, @vm_deleter, @disk_manager, job_renderer)
+          arp_flusher = ArpFlusher.new
+          @vm_creator = Bosh::Director::VmCreator.new(@cloud, @logger, @vm_deleter, @disk_manager, job_renderer, arp_flusher)
         end
 
         def perform
@@ -36,7 +36,7 @@ module Bosh::Director
 
           @logger.info('Creating missing VMs')
           # TODO: something about instance_plans.select(&:new?) -- how does that compare to the isntance#has_vm? check?
-          @vm_creator.create_for_instance_plans(@deployment_plan.instance_plans_with_missing_vms, @deployment_plan.ip_provider, @event_log)
+          @vm_creator.create_for_instance_plans(@deployment_plan.instance_plans_with_missing_vms, @deployment_plan.ip_provider)
 
           @base_job.task_checkpoint
         end
@@ -56,7 +56,7 @@ module Bosh::Director
             @logger.info('No unneeded instances to delete')
             return
           end
-          event_log_stage = @event_log.begin_stage('Deleting unneeded instances', unneeded_instances.size)
+          event_log_stage = Config.event_log.begin_stage('Deleting unneeded instances', unneeded_instances.size)
           dns_manager = DnsManagerProvider.create
           instance_deleter = InstanceDeleter.new(@deployment_plan.ip_provider, dns_manager, @disk_manager)
           unneeded_instance_plans = unneeded_instances.map do |instance|

data/lib/bosh/director/deployment_plan/template.rb

@@ -124,7 +124,7 @@ module Bosh::Director
         end
       end
 
-      def add_link_info(job_name, kind, link_name, source)
+      def add_link_from_release(job_name, kind, link_name, source)
         @link_infos[job_name] ||= {}
         @link_infos[job_name][kind] ||= {}
         @link_infos[job_name][kind][link_name] ||= {}
@@ -136,6 +136,37 @@ module Bosh::Director
           @link_infos[job_name][kind][link_name]['skip_link'] = true
         else
           source.to_a.each do |key, value|
+            if key == "properties"
+              key = "link_properties_exported"
+            end
+            @link_infos[job_name][kind][link_name][key] = value
+          end
+        end
+      end
+
+      def add_link_from_manifest(job_name, kind, link_name, source)
+        @link_infos[job_name] ||= {}
+        @link_infos[job_name][kind] ||= {}
+        @link_infos[job_name][kind][link_name] ||= {}
+
+        if source.eql? 'nil'
+          # This is the case where the user set link source to nil explicitly in the deployment manifest
+          # We should skip the binding of this link, even if it exist. This is used only when the link
+          # is optional
+          @link_infos[job_name][kind][link_name]['skip_link'] = true
+        else
+          errors = []
+          if kind == "consumes"
+            errors = validate_consume_link(source, link_name, job_name)
+          end
+          errors.concat(validate_link_def(source, link_name, job_name))
+
+          if errors.size > 0
+            raise errors.join("\n")
+          end
+
+          source_hash = source.to_a
+          source_hash.each do |key, value|
             @link_infos[job_name][kind][link_name][key] = value
           end
         end
@@ -177,6 +208,35 @@ module Bosh::Director
 
       private
 
+      def validate_consume_link(source, link_name, job_name)
+        blacklist = [ ['instances', 'from'], ['properties', 'from'] ]
+        errors = []
+        if source == nil
+          return errors
+        end
+
+        blacklist.each do |invalid_props|
+          if invalid_props.all? { |prop| source.has_key?(prop) }
+            errors.push("Cannot specify both '#{invalid_props[0]}' and '#{invalid_props[1]}' keys for link '#{link_name}' in job '#{@name}' in instance group '#{job_name}'.")
+          end
+        end
+
+        if source.has_key?('properties') && !source.has_key?('instances')
+          errors.push("Cannot specify 'properties' without 'instances' for link '#{link_name}' in job '#{@name}' in instance group '#{job_name}'.")
+        end
+
+        errors
+      end
+
+      def validate_link_def(source, link_name, job_name)
+        errors = []
+        if !source.nil? && (source.has_key?('name') || source.has_key?('type'))
+          errors.push("Cannot specify 'name' or 'type' properties in the manifest for link '#{link_name}' in job '#{@name}' in instance group '#{job_name}'. Please provide these keys in the release only.")
+        end
+
+        errors
+      end
+
       # Returns model only if it's present, fails otherwise
       # @return [Models::Template]
       def present_model