bosh-director 1.3202.0 → 1.3213.0

data/lib/bosh/director/deployment_plan/manifest_validator.rb

@@ -15,7 +15,7 @@ module Bosh
               manifest['jobs'].each do |job|
                 if job.has_key?('migrated_from')
                   raise Bosh::Director::DeploymentInvalidProperty,
-                    "Deployment manifest jobs contain 'migrated_from', but it can only be used with cloud-config."
+                    "Deployment manifest instance groups contain 'migrated_from', but it can only be used with cloud-config."
                 end
               end
             end

data/lib/bosh/director/deployment_plan/network_settings.rb

@@ -1,15 +1,15 @@
 module Bosh::Director::DeploymentPlan
   class NetworkSettings
-    def initialize(job_name, deployment_name, default_network, desired_reservations, state, availability_zone, instance_index, instance_id, dns_manager)
+    def initialize(job_name, deployment_name, default_network, desired_reservations, current_networks, availability_zone, instance_index, instance_id, dns_manager)
       @job_name = job_name
       @desired_reservations = desired_reservations
       @default_network = default_network
       @deployment_name = deployment_name
-      @state = state
       @availability_zone = availability_zone
       @instance_index = instance_index
       @instance_id = instance_id
       @dns_manager = dns_manager
+      @current_networks = current_networks
     end
 
     def to_hash
@@ -22,16 +22,12 @@ module Bosh::Director::DeploymentPlan
       @desired_reservations.each do |reservation|
         network_name = reservation.network.name
         network_settings[network_name] = reservation.network.network_settings(reservation, default_properties[network_name], @availability_zone)
-
         # Somewhat of a hack: for dynamic networks we might know IP address, Netmask & Gateway
         # if they're featured in agent state, in that case we put them into network spec to satisfy
         # ConfigurationHasher in both agent and director.
-        if @state.is_a?(Hash) &&
-          @state['networks'].is_a?(Hash) &&
-          @state['networks'][network_name].is_a?(Hash) &&
-          network_settings[network_name]['type'] == 'dynamic'
+        if @current_networks.is_a?(Hash) && @current_networks[network_name].is_a?(Hash) && network_settings[network_name]['type'] == 'dynamic'
           %w(ip netmask gateway).each do |key|
-            network_settings[network_name][key] = @state['networks'][network_name][key]
+            network_settings[network_name][key] = @current_networks[network_name][key] unless @current_networks[network_name][key].nil?
           end
         end
       end
@@ -50,16 +46,34 @@ module Bosh::Director::DeploymentPlan
       dns_record_info
     end
 
+    def network_address(preferred_network_name = nil)
+      network_name = preferred_network_name || @default_network['gateway']
+      network_hash = to_hash
+
+      if network_hash[network_name]['type'] == 'dynamic'
+        address = @dns_manager.dns_record_name(@instance_id, @job_name, network_name, @deployment_name)
+      else
+        address = network_hash[network_name]['ip']
+      end
+
+      address
+    end
+
     def network_addresses
       network_addresses = {}
+
       to_hash.each do |network_name, network|
-        network_addresses[network_name] = {
-          'address' => network['type'] == 'dynamic' ?
-            @dns_manager.dns_record_name(@instance_id, @job_name, network_name, @deployment_name) :
-            network['ip']
-        }
+        if network['type'] == 'dynamic'
+          address = @dns_manager.dns_record_name(@instance_id, @job_name, network_name, @deployment_name)
+        else
+          address = network['ip']
+        end
+
+        network_addresses[network_name] = address
       end
+
       network_addresses
     end
+
   end
 end

data/lib/bosh/director/deployment_plan/package_validator.rb

@@ -6,19 +6,23 @@ module Bosh::Director
         @logger = logger
       end
 
-      def validate(release_version_model, stemcel_model)
+      def validate(release_version_model, stemcell_model)
         release_desc = "#{release_version_model.release.name}/#{release_version_model.version}"
 
         @logger.debug("Validating packages for release '#{release_desc}'")
         release_version_model.packages.each do |package|
-          packages_list = release_version_model.transitive_dependencies(package)
+          packages_list = Bosh::Director::PackageDependenciesManager.new(release_version_model).transitive_dependencies(package)
           packages_list << package
           packages_list.each do |needed_package|
             if needed_package.sha1.nil? || needed_package.blobstore_id.nil?
-              compiled_packages_list = Bosh::Director::Models::CompiledPackage[:package_id => needed_package.id, :stemcell_id => stemcel_model.id]
-              if compiled_packages_list.nil?
+              compiled_packages_list = Bosh::Director::Models::CompiledPackage.where(:package_id => needed_package.id,
+                :stemcell_os => stemcell_model.operating_system).all
+              compiled_packages_list = compiled_packages_list.select do |compiled_package|
+                Bosh::Common::Version::StemcellVersion.match(compiled_package.stemcell_version, stemcell_model.version)
+              end
+              if compiled_packages_list.empty?
                 @faults[release_desc] ||= Set.new
-                @faults[release_desc] << Fault.new(needed_package, stemcel_model)
+                @faults[release_desc] << Fault.new(needed_package, stemcell_model)
               end
             end
           end

data/lib/bosh/director/deployment_plan/placement_planner/networks_to_static_ips.rb

@@ -19,7 +19,7 @@ module Bosh
                 subnet_for_ip = subnets.find { |subnet| subnet.static_ips.include?(static_ip) }
                 if subnet_for_ip.nil?
                   raise JobNetworkInstanceIpMismatch,
-                    "Job '#{job_name}' with network '#{job_network.name}' declares static ip '#{format_ip(static_ip)}', " +
+                    "Instance group '#{job_name}' with network '#{job_network.name}' declares static ip '#{format_ip(static_ip)}', " +
                       "which belongs to no subnet"
                 end
                 az_names = subnet_for_ip.availability_zone_names.nil? ? [nil] : subnet_for_ip.availability_zone_names
@@ -44,7 +44,7 @@ module Bosh
               end
 
               raise JobInvalidAvailabilityZone,
-                "Job '#{@job_name}' subnets declare availability zones and the job does not"
+                "Instance group '#{@job_name}' subnets declare availability zones and the instance group does not"
             end
           end
 
@@ -59,7 +59,7 @@ module Bosh
 
               if non_desired_ip_to_az
                 raise JobStaticIpsFromInvalidAvailabilityZone,
-                  "Job '#{@job_name}' declares static ip '#{format_ip(non_desired_ip_to_az.ip)}' which does not belong to any of the job's availability zones."
+                  "Instance group '#{@job_name}' declares static ip '#{format_ip(non_desired_ip_to_az.ip)}' which does not belong to any of the instance group's availability zones."
               end
             end
           end
@@ -79,7 +79,7 @@ module Bosh
           def distribute_evenly_per_zone
             best_combination = BruteForceIpAllocation.new(@networks_to_static_ips).find_best_combination
             if best_combination.nil?
-              raise JobNetworkInstanceIpMismatch, "Failed to evenly distribute static IPs between zones for job '#{@job_name}'"
+              raise JobNetworkInstanceIpMismatch, "Failed to evenly distribute static IPs between zones for instance group '#{@job_name}'"
             end
             @networks_to_static_ips = best_combination
           end

data/lib/bosh/director/deployment_plan/planner.rb

@@ -1,5 +1,6 @@
 require 'bosh/director/deployment_plan/deployment_spec_parser'
 require 'bosh/director/deployment_plan/cloud_manifest_parser'
+require 'bosh/director/deployment_plan/runtime_manifest_parser'
 require 'bosh/director/deployment_plan/disk_type'
 require 'forwardable'
 require 'common/deep_copy'
@@ -51,15 +52,14 @@ module Bosh::Director
       # @return [Boolean] Indicates whether VMs should be drained
       attr_reader :skip_drain
 
-      def initialize(attrs, manifest_text, cloud_config, deployment_model, options = {})
-        @cloud_config = cloud_config
-
+      def initialize(attrs, manifest_text, cloud_config, runtime_config, deployment_model, options = {})
         @name = attrs.fetch(:name)
         @properties = attrs.fetch(:properties)
         @releases = {}
 
         @manifest_text = Bosh::Common::DeepCopy.copy(manifest_text)
         @cloud_config = cloud_config
+        @runtime_config = runtime_config
         @model = deployment_model
 
         @stemcells = {}
@@ -88,6 +88,8 @@ module Bosh::Director
         :resource_pool,
         :vm_types,
         :vm_type,
+        :vm_extensions,
+        :vm_extension,
         :add_resource_pool,
         :disk_types,
         :disk_type,
@@ -98,7 +100,7 @@ module Bosh::Director
         Canonicalizer.canonicalize(@name)
       end
 
-      def bind_models
+      def bind_models(skip_links_binding = false)
         stemcell_manager = Api::StemcellManager.new
         dns_manager = DnsManagerProvider.create
         assembler = DeploymentPlan::Assembler.new(
@@ -109,7 +111,7 @@ module Bosh::Director
           @logger
         )
 
-        assembler.bind_models
+        assembler.bind_models(skip_links_binding)
       end
 
       def compile_packages
@@ -209,7 +211,7 @@ module Bosh::Director
       def add_job(job)
         if @jobs_canonical_name_index.include?(job.canonical_name)
           raise DeploymentCanonicalJobNameTaken,
-            "Invalid job name `#{job.name}', canonical name already taken"
+            "Invalid instance group name '#{job.name}', canonical name already taken"
         end
 
         @jobs << job
@@ -225,7 +227,19 @@ module Bosh::Director
       end
 
       def jobs_starting_on_deploy
-        @jobs.select(&:starts_on_deploy?)
+        jobs = []
+
+        @jobs.each do |job|
+          if job.is_service?
+            jobs << job
+          elsif job.is_errand?
+            if job.instances.any? { |i| nil != i.model && !i.model.vm_cid.to_s.empty? }
+              jobs << job
+            end
+          end
+        end
+
+        jobs
       end
 
       def persist_updates!
@@ -241,6 +255,7 @@ module Bosh::Director
 
         model.manifest = Psych.dump(@manifest_text)
         model.cloud_config = @cloud_config
+        model.runtime_config = @runtime_config
         model.link_spec = @link_spec
         model.save
       end
@@ -284,6 +299,7 @@ module Bosh::Director
         @global_network_resolver = options.fetch(:global_network_resolver)
         @resource_pools = self.class.index_by_name(options.fetch(:resource_pools))
         @vm_types = self.class.index_by_name(options.fetch(:vm_types, {}))
+        @vm_extensions = self.class.index_by_name(options.fetch(:vm_extensions, {}))
         @disk_types = self.class.index_by_name(options.fetch(:disk_types))
         @availability_zones = options.fetch(:availability_zones_list)
         @compilation = options.fetch(:compilation)
@@ -332,6 +348,14 @@ module Bosh::Director
         @vm_types[name]
       end
 
+      def vm_extensions
+        @vm_extensions.values
+      end
+
+      def vm_extension(name)
+        @vm_extensions[name]
+      end
+
       def add_resource_pool(resource_pool)
         @resource_pools[resource_pool.name] = resource_pool
       end

data/lib/bosh/director/deployment_plan/planner_factory.rb

@@ -33,17 +33,17 @@ module Bosh
         end
 
         def create_from_model(deployment_model, options={})
-          manifest = Manifest.load_from_text(deployment_model.manifest, deployment_model.cloud_config)
-          create_from_manifest(manifest, deployment_model.cloud_config, options)
+          manifest = Manifest.load_from_text(deployment_model.manifest, deployment_model.cloud_config, deployment_model.runtime_config)
+          create_from_manifest(manifest, deployment_model.cloud_config, deployment_model.runtime_config, options)
         end
 
-        def create_from_manifest(manifest, cloud_config, options)
-          parse_from_manifest(manifest, cloud_config, options)
+        def create_from_manifest(manifest, cloud_config, runtime_config, options)
+          parse_from_manifest(manifest, cloud_config, runtime_config, options)
         end
 
         private
 
-        def parse_from_manifest(manifest, cloud_config, options)
+        def parse_from_manifest(manifest, cloud_config, runtime_config, options)
           manifest.resolve_aliases
           @manifest_validator.validate(manifest.manifest_hash, manifest.cloud_config_hash)
           deployment_manifest, cloud_manifest = @deployment_manifest_migrator.migrate(manifest.manifest_hash, manifest.cloud_config_hash)
@@ -67,15 +67,156 @@ module Bosh
           @logger.info('Creating deployment plan')
           @logger.info("Deployment plan options: #{plan_options}")
 
-          deployment = Planner.new(attrs, deployment_manifest, cloud_config, deployment_model, plan_options)
+          deployment = Planner.new(attrs, deployment_manifest, cloud_config, runtime_config, deployment_model, plan_options)
           global_network_resolver = GlobalNetworkResolver.new(deployment)
 
           ip_provider_factory = IpProviderFactory.new(deployment.using_global_networking?, @logger)
           deployment.cloud_planner = CloudManifestParser.new(@logger).parse(cloud_manifest, global_network_resolver, ip_provider_factory)
           DeploymentSpecParser.new(deployment, Config.event_log, @logger).parse(deployment_manifest, plan_options)
+
+          if runtime_config
+            RuntimeManifestParser.new(@logger, deployment).parse(runtime_config.manifest)
+          end
+
+          process_links(deployment)
+
           DeploymentValidator.new.validate(deployment)
           deployment
         end
+
+        def process_links(deployment)
+          errors = []
+
+          deployment.jobs.each do |current_job|
+            current_job.templates.each do |template|
+              if template.link_infos.has_key?(current_job.name) && template.link_infos[current_job.name].has_key?('consumes')
+                template.link_infos[current_job.name]['consumes'].each do |name, source|
+                  link_path = LinkPath.new(deployment, current_job.name, template.name)
+
+                  begin
+                    link_path.parse(source)
+                  rescue Exception => e
+                    errors.push e
+                  end
+
+                  if !link_path.skip
+                    current_job.add_link_path(template.name, name, link_path)
+                  end
+                end
+              end
+
+              ## Choose between using template-scoped and other props.
+              ## if job manifest had a "properties key" in the template block
+              if template.template_scoped_properties.has_key?(current_job.name)
+                scoped_properties = template.template_scoped_properties[current_job.name]
+              else
+                scoped_properties = current_job.all_properties || {}
+              end
+
+              if template.link_infos.has_key?(current_job.name) && template.link_infos[current_job.name].has_key?('provides')
+                template.link_infos[current_job.name]['provides'].each do |link_name, provided_link|
+                  if provided_link['properties']
+                    ## Get default values for this job
+                    default_properties = get_default_properties(deployment, template)
+
+                    provided_link['mapped_properties'] = process_link_properties(scoped_properties, default_properties, provided_link['properties'], errors)
+                  end
+                end
+              end
+            end
+          end
+
+          if errors.length > 0
+            message = 'Unable to process links for deployment. Errors are:'
+
+            errors.each do |e|
+              message = "#{message}\n   - \"#{e.message.gsub(/\n/, "\n  ")}\""
+            end
+
+            raise message
+          end
+        end
+
+        def get_default_properties(deployment, template)
+          release_manager = Api::ReleaseManager.new
+
+          release_versions_templates_models_hash = {}
+
+          template_name = template.name
+          release_name = template.release.name
+
+          release = deployment.release(release_name)
+
+          if !release_versions_templates_models_hash.has_key?(release_name)
+            release_model = release_manager.find_by_name(release_name)
+            current_release_version = release_manager.find_version(release_model, release.version)
+            release_versions_templates_models_hash[release_name] = current_release_version.templates
+          end
+
+          templates_models_list = release_versions_templates_models_hash[release_name]
+          current_template_model = templates_models_list.find {|target| target.name == template_name }
+
+          if current_template_model.properties != nil
+            default_prop = {}
+            default_prop['properties'] = current_template_model.properties
+            default_prop["template_name"] = template.name
+            return default_prop
+          end
+
+          return {"template_name" => template.name}
+        end
+
+        def process_link_properties(scoped_properties, default_properties, link_property_list, errors)
+          mapped_properties = {}
+
+          link_property_list.each do |link_property| #list of properties
+            previous_property_in_loop = {}
+            current_property_in_loop = scoped_properties
+            mapped_properties_in_loop = mapped_properties
+
+            use_defaults = false
+            property_path = link_property.split(".")
+            property_path.each do |key|
+              if !current_property_in_loop || !current_property_in_loop.has_key?(key)
+                use_defaults = true
+              else
+                current_property_in_loop = current_property_in_loop[key]
+              end
+
+              if !mapped_properties_in_loop.has_key?(key)
+                mapped_properties_in_loop[key] = {}
+              end
+              previous_property_in_loop = mapped_properties_in_loop
+              mapped_properties_in_loop = mapped_properties_in_loop[key]
+            end
+
+            if use_defaults
+              if default_properties.has_key?('properties') && default_properties['properties'].has_key?(link_property)
+                if default_properties['properties'][link_property].has_key?('default')
+                  previous_property_in_loop[property_path.last()] = default_properties['properties'][link_property]['default']
+                else
+                  e = Exception.new("Link property #{link_property} in template #{default_properties['template_name']} has no default value or value supplied by the deployment manifest")
+                  errors.push(e)
+                end
+              else
+                e = Exception.new("Link property #{link_property} in template #{default_properties['template_name']} is not defined in release spec")
+                errors.push(e)
+              end
+            else
+              previous_property_in_loop[property_path.last()] = current_property_in_loop
+            end
+
+            # if use_defaults && !default_properties.has_key?("properties") && !default_properties['properties'][link_property].has_key?('default')
+            #   e = Exception.new("Property #{link_property} in template #{default_properties['template_name']} has no default value or value supplied by the deployment manifest")
+            #   errors.push(e)
+            # elsif use_defaults
+            #   previous_property_in_loop[property_path.last()] = default_properties['properties'][link_property]['default']
+            # else
+            #   previous_property_in_loop[property_path.last()] = current_property_in_loop
+            # end
+          end
+          return mapped_properties
+        end
       end
     end
   end

data/lib/bosh/director/deployment_plan/runtime_manifest_parser.rb

@@ -0,0 +1,142 @@
+module Bosh::Director
+  module DeploymentPlan
+    class RuntimeManifestParser
+      include ValidationHelper
+
+      def initialize(logger, deployment=nil)
+        @deployment = deployment
+        @logger = logger
+      end
+
+      def parse(runtime_manifest)
+        parse_releases(runtime_manifest)
+        parse_addons(runtime_manifest)
+      end
+
+      private
+
+      def parse_releases(runtime_manifest)
+        @release_specs = []
+
+        if runtime_manifest['release']
+          if runtime_manifest['releases']
+            raise RuntimeAmbiguousReleaseSpec,
+                  "Runtime manifest contains both `release' and `releases' " +
+                      'sections, please use one of the two.'
+          end
+          @release_specs << runtime_manifest['release']
+        else
+          safe_property(runtime_manifest, 'releases', :class => Array).each do |release|
+            @release_specs << release
+          end
+        end
+
+        @release_specs.each do |release_spec|
+          if release_spec['version'] == 'latest'
+            raise RuntimeInvalidReleaseVersion,
+                  "Runtime manifest contains the release `#{release_spec['name']}' with version as `latest'. " +
+                      "Please specify the actual version string."
+          end
+
+          if @deployment
+            deployment_release = @deployment.release(release_spec["name"])
+            if deployment_release
+              if deployment_release.version != release_spec["version"]
+                raise RuntimeInvalidDeploymentRelease, "Runtime manifest specifies release `#{release_spec["name"]}' with version as `#{release_spec["version"]}'. " +
+                      "This conflicts with version `#{deployment_release.version}' specified in the deployment manifest."
+              else
+                next
+              end
+            end
+
+            release_version = DeploymentPlan::ReleaseVersion.new(@deployment.model, release_spec)
+            release_version.bind_model
+
+            @deployment.add_release(release_version)
+          end
+        end
+      end
+
+      def parse_addons(runtime_manifest)
+        addons = safe_property(runtime_manifest, 'addons', :class => Array, :default => [])
+        addons.each do |addon_spec|
+          deployment_plan_templates = []
+
+          addon_jobs = safe_property(addon_spec, 'jobs', :class => Array, :default => [])
+
+          addon_jobs.each do |job|
+            if !@release_specs.find { |release_spec| release_spec['name'] == job['release'] }
+              raise RuntimeReleaseNotListedInReleases,
+                    "Runtime manifest specifies job `#{job['name']}' which is defined in `#{job['release']}', but `#{job['release']}' is not listed in the releases section."
+            end
+
+            if @deployment
+              valid_release_versions = @deployment.releases.map {|r| r.name }
+              deployment_release_ids = Models::Release.where(:name => valid_release_versions).map {|r| r.id}
+              deployment_jobs = @deployment.jobs
+
+              templates_from_model = Models::Template.where(:name => job['name'], :release_id => deployment_release_ids)
+              if templates_from_model == nil
+                raise "Job '#{job['name']}' not found in Template table"
+              end
+
+              release = @deployment.release(job['release'])
+              release.bind_model
+
+              template = DeploymentPlan::Template.new(release, job['name'])
+
+              deployment_jobs.each do |j|
+                templates_from_model.each do |template_from_model|
+                  if template_from_model.consumes != nil
+                    template_from_model.consumes.each do |consumes|
+                      template.add_link_info(j.name, 'consumes', consumes["name"], consumes)
+                    end
+                  end
+                  if template_from_model.provides != nil
+                    template_from_model.provides.each do |provides|
+                      template.add_link_info(j.name, 'provides', provides["name"], provides)
+                    end
+                  end
+                end
+
+                provides_links = safe_property(job, 'provides', class: Hash, optional: true)
+                provides_links.to_a.each do |link_name, source|
+                  template.add_link_info(j.name, "provides", link_name, source)
+                end
+
+                consumes_links = safe_property(job, 'consumes', class: Hash, optional: true)
+                consumes_links.to_a.each do |link_name, source|
+                  template.add_link_info(j.name, 'consumes', link_name, source)
+                end
+              end
+
+              template.bind_models
+              deployment_plan_templates.push(template)
+
+              deployment_jobs.each do |job|
+                merge_addon(job, deployment_plan_templates, addon_spec['properties'])
+              end
+
+            end
+          end
+        end
+      end
+
+      def merge_addon(job, templates, properties)
+        if job.templates
+          job.templates.concat(templates)
+        else
+          job.templates = templates
+        end
+
+        if properties
+          if job.all_properties
+            job.all_properties.merge!(properties)
+          else
+            job.all_properties = properties
+          end
+        end
+      end
+    end
+  end
+end