bosh-director 1.3202.0 → 1.3213.0

data/lib/bosh/director/compile_task_generator.rb

@@ -11,7 +11,7 @@ module Bosh::Director
     def generate!(compile_tasks, job, template, package, stemcell)
       # Our assumption here is that package dependency graph
       # has no cycles: this is being enforced on release upload.
-      # Other than that it's a vanilla DFS.
+      # Other than that it's a vanilla Depth-First Search (DFS).
 
       @logger.info("Checking whether package `#{package.desc}' needs to be compiled for stemcell `#{stemcell.model.desc}'")
       task_key = [package.id, stemcell.id]
@@ -23,10 +23,10 @@ module Bosh::Director
       end
 
       release_version = template.release.model
-
-      transitive_dependencies = release_version.transitive_dependencies(package)
-      package_dependency_key = Models::CompiledPackage.create_dependency_key(transitive_dependencies)
-      package_cache_key = Models::CompiledPackage.create_cache_key(package, transitive_dependencies, stemcell.model)
+      package_dependency_manager = PackageDependenciesManager.new(release_version)
+      transitive_dependencies = package_dependency_manager.transitive_dependencies(package)
+      package_dependency_key = KeyGenerator.new.dependency_key_from_models(package, release_version)
+      package_cache_key = Models::CompiledPackage.create_cache_key(package, transitive_dependencies, stemcell.model.sha1)
 
       task = CompileTask.new(package, stemcell, job, package_dependency_key, package_cache_key)
 
@@ -36,7 +36,7 @@ module Bosh::Director
       end
 
       @logger.info("Processing package `#{package.desc}' dependencies")
-      dependencies = release_version.dependencies(package)
+      dependencies = package_dependency_manager.dependencies(package)
       dependencies.each do |dependency|
         @logger.info("Package `#{package.desc}' depends on package `#{dependency.desc}'")
         dependency_task = generate!(compile_tasks, job, template, dependency, stemcell)

data/lib/bosh/director/compiled_package_group.rb

@@ -7,11 +7,12 @@ module Bosh::Director
 
     def compiled_packages
       @compiled_packages ||= @release_version.packages.map do |package|
-        transitive_dependencies = @release_version.transitive_dependencies(package)
-        package_dependency_key = Bosh::Director::Models::CompiledPackage.create_dependency_key(transitive_dependencies)
+        package_dependency_key = KeyGenerator.new.dependency_key_from_models(package, @release_version)
+
         Models::CompiledPackage[
           :package_id => package.id,
-          :stemcell_id => @stemcell.id,
+          :stemcell_os => @stemcell.operating_system,
+          :stemcell_version => @stemcell.version,
           :dependency_key => package_dependency_key
         ]
       end.compact

data/lib/bosh/director/compiled_release.rb

@@ -0,0 +1,6 @@
+module Bosh::Director
+  module CompiledRelease
+  end
+end
+
+require 'bosh/director/compiled_release/manifest'

data/lib/bosh/director/compiled_release/manifest.rb

@@ -0,0 +1,30 @@
+module Bosh::Director
+  module CompiledRelease
+    class Manifest
+      def initialize(manifest_hash)
+        @manifest = manifest_hash
+      end
+
+      def has_matching_package(package_name, stemcell_os, stemcell_version, dependency_key)
+
+        "#{stemcell_os}/#{stemcell_version}" == stemcell_os_and_version(package_name) &&
+            dependency_key == dependency_key(package_name)
+      end
+
+      def dependency_key(package_name)
+        KeyGenerator.new.dependency_key_from_manifest(package_name, @manifest['compiled_packages'])
+      end
+
+      private
+
+      def meta_data(package_name)
+        @manifest['compiled_packages'].find { |p| p['name'] == package_name }
+      end
+
+      def stemcell_os_and_version(package_name)
+        meta_data = meta_data(package_name)
+        meta_data.nil? ? nil : meta_data['stemcell']
+      end
+    end
+  end
+end

data/lib/bosh/director/compiled_release_manifest.rb

@@ -42,11 +42,11 @@ module Bosh::Director
 
     def get_dependencies(compiled_package)
       dependencies = []
-      parser = Yajl::Parser.new
-      hash = parser.parse(compiled_package.package.dependency_set_json)
-      hash.each do |name|
+
+      compiled_package.package.dependency_set.each do |name|
         dependencies << name
       end
+
       dependencies
     end
 

data/lib/bosh/director/config.rb

@@ -33,7 +33,10 @@ module Bosh::Director
         :max_vm_create_tries,
         :nats_uri,
         :default_ssh_options,
-        :keep_unreachable_vms
+        :keep_unreachable_vms,
+        :enable_post_deploy,
+        :generate_vm_passwords,
+        :remove_dev_tools,
       )
 
       attr_reader(
@@ -163,6 +166,9 @@ module Bosh::Director
         @ignore_missing_gateway = config['ignore_missing_gateway']
 
         @keep_unreachable_vms = config.fetch('keep_unreachable_vms', false)
+        @enable_post_deploy = config.fetch('enable_post_deploy', false)
+        @generate_vm_passwords = config.fetch('generate_vm_passwords', false)
+        @remove_dev_tools = config['remove_dev_tools']
 
         Bosh::Clouds::Config.configure(self)
 
@@ -458,6 +464,10 @@ module Bosh::Director
       Config.configure(hash)
     end
 
+    def get_uuid_provider
+      Bosh::Director::Api::DirectorUUIDProvider.new(Config)
+    end
+
     private
 
     attr_reader :hash

data/lib/bosh/director/deployment_plan.rb

@@ -22,6 +22,7 @@ require 'bosh/director/deployment_plan/release_version'
 require 'bosh/director/deployment_plan/resource_pool'
 require 'bosh/director/deployment_plan/env'
 require 'bosh/director/deployment_plan/vm_type'
+require 'bosh/director/deployment_plan/vm_extension'
 require 'bosh/director/deployment_plan/stemcell'
 require 'bosh/director/deployment_plan/template'
 require 'bosh/director/deployment_plan/update_config'

data/lib/bosh/director/deployment_plan/assembler.rb

@@ -13,7 +13,7 @@ module Bosh::Director
       @dns_manager = dns_manager
     end
 
-    def bind_models
+    def bind_models(skip_links_binding = false)
       @logger.info('Binding models')
       bind_releases
 
@@ -43,7 +43,11 @@ module Bosh::Director
       bind_properties
       bind_instance_networks
       bind_dns
-      bind_links
+
+      if (!skip_links_binding)
+        bind_links
+      end
+
     end
 
     private

data/lib/bosh/director/deployment_plan/cloud_manifest_parser.rb

@@ -13,8 +13,9 @@ module Bosh::Director
         networks = parse_networks(cloud_manifest, global_network_resolver, azs)
         resource_pools = parse_resource_pools(cloud_manifest)
         vm_types = parse_vm_types(cloud_manifest)
+        vm_extensions = parse_vm_extensions(cloud_manifest)
         disk_types = parse_disk_types(cloud_manifest)
-        compilation_config = parse_compilation(cloud_manifest, networks, az_list, vm_types)
+        compilation_config = parse_compilation(cloud_manifest, networks, az_list, vm_types, vm_extensions)
 
         CloudPlanner.new({
           availability_zones_list: az_list,
@@ -24,6 +25,7 @@ module Bosh::Director
           compilation: compilation_config,
           resource_pools: resource_pools,
           vm_types: vm_types,
+          vm_extensions: vm_extensions,
           disk_types: disk_types,
           logger: @logger,
         })
@@ -39,7 +41,7 @@ module Bosh::Director
 
         duplicates = detect_duplicates(parsed_availability_zones) { |az| az.name }
         unless duplicates.empty?
-          raise DeploymentDuplicateAvailabilityZoneName, "Duplicate az name `#{duplicates.first.name}'"
+          raise DeploymentDuplicateAvailabilityZoneName, "Duplicate az name '#{duplicates.first.name}'"
         end
 
         parsed_availability_zones
@@ -63,27 +65,27 @@ module Bosh::Director
               VipNetwork.new(network_spec, @logger)
             else
               raise DeploymentInvalidNetworkType,
-                "Invalid network type `#{type}'"
+                "Invalid network type '#{type}'"
           end
         end
 
         duplicates = detect_duplicates(parsed_networks) { |network| network.canonical_name }
         unless duplicates.empty?
-          raise DeploymentCanonicalNetworkNameTaken, "Invalid network name `#{duplicates.first.name}', canonical name already taken"
+          raise DeploymentCanonicalNetworkNameTaken, "Invalid network name '#{duplicates.first.name}', canonical name already taken"
         end
 
         parsed_networks
       end
 
-      def parse_compilation(cloud_manifest, networks, az_list, vm_types)
+      def parse_compilation(cloud_manifest, networks, az_list, vm_types, vm_extensions)
         compilation_spec = safe_property(cloud_manifest, 'compilation', :class => Hash)
-        config = CompilationConfig.new(compilation_spec, az_list, vm_types)
+        config = CompilationConfig.new(compilation_spec, az_list, vm_types, vm_extensions)
 
         compilation_network = networks.find { |network| network.name == config.network_name }
         if compilation_network.nil?
           raise CompilationConfigUnknownNetwork,
             "Compilation config references an unknown " +
-              "network `#{config.network_name}'"
+              "network '#{config.network_name}'"
         end
 
         unless compilation_network.has_azs?([config.availability_zone_name])
@@ -111,7 +113,7 @@ module Bosh::Director
 
         duplicates = detect_duplicates(parsed_resource_pools) { |rp| rp.name }
         unless duplicates.empty?
-          raise DeploymentDuplicateResourcePoolName, "Duplicate resource pool name `#{duplicates.first.name}'"
+          raise DeploymentDuplicateResourcePoolName, "Duplicate resource pool name '#{duplicates.first.name}'"
         end
 
         parsed_resource_pools
@@ -126,12 +128,26 @@ module Bosh::Director
 
         duplicates = detect_duplicates(parsed_vm_types) { |vmt| vmt.name }
         unless duplicates.empty?
-          raise DeploymentDuplicateVmTypeName, "Duplicate vm type name `#{duplicates.first.name}'"
+          raise DeploymentDuplicateVmTypeName, "Duplicate vm type name '#{duplicates.first.name}'"
         end
 
         parsed_vm_types
       end
 
+      def parse_vm_extensions(cloud_manifest)
+        vm_extensions = safe_property(cloud_manifest, 'vm_extensions', :class => Array, :optional => true, :default => [])
+
+        parsed_vm_extensions = vm_extensions.map do |vmt_spec|
+          VmExtension.new(vmt_spec)
+        end
+
+        duplicates = detect_duplicates(parsed_vm_extensions) { |vmt| vmt.name }
+        unless duplicates.empty?
+          raise DeploymentDuplicateVmExtensionName, "Duplicate vm extension name '#{duplicates.first.name}'"
+        end
+
+        parsed_vm_extensions
+      end
 
       def parse_disk_types(cloud_manifest)
         disk_pools = safe_property(cloud_manifest, 'disk_pools', :class => Array, :optional => true, :default => [])
@@ -158,7 +174,7 @@ module Bosh::Director
 
         duplicates = detect_duplicates(parsed_disk_types) { |dp| dp.name }
         unless duplicates.empty?
-          raise DeploymentDuplicateDiskTypeName, "Duplicate disk #{disk_source} name `#{duplicates.first.name}'"
+          raise DeploymentDuplicateDiskTypeName, "Duplicate disk #{disk_source} name '#{duplicates.first.name}'"
         end
 
         parsed_disk_types

data/lib/bosh/director/deployment_plan/compilation_config.rb

@@ -23,7 +23,9 @@ module Bosh::Director
 
       attr_reader :vm_type
 
-      def initialize(compilation_config, azs_list, vm_types = [])
+      attr_reader :vm_extensions
+
+      def initialize(compilation_config, azs_list, vm_types = [], vm_extensions = [])
         @workers = safe_property(compilation_config, 'workers', class: Integer, min: 1)
 
         @network_name = safe_property(compilation_config, 'network', class: String)
@@ -37,33 +39,67 @@ module Bosh::Director
           compilation_config, 'cloud_properties', class: Hash, default: {})
         @env = safe_property(compilation_config, 'env', class: Hash, optional: true, default: {})
 
+        parse_availability_zone(azs_list, compilation_config)
+
+        parse_vm_type(compilation_config, vm_types)
+
+        parse_vm_extensions(compilation_config, vm_extensions)
+
+      end
+
+      def availability_zone_name
+        @availability_zone.nil? ? nil : @availability_zone.name
+      end
+
+      private
+
+      def parse_availability_zone(azs_list, compilation_config)
         az_name = safe_property(compilation_config, 'az', class: String, optional: true)
         @availability_zone = azs_list[az_name]
         if az_name && !az_name.empty? && @availability_zone.nil?
           raise Bosh::Director::CompilationConfigInvalidAvailabilityZone,
-            "Compilation config references unknown az '#{az_name}'. Known azs are: [#{azs_list.keys.join(', ')}]"
+                "Compilation config references unknown az '#{az_name}'. Known azs are: [#{azs_list.keys.join(', ')}]"
         end
+      end
 
+      def parse_vm_type(compilation_config, vm_types)
         vm_type_name = safe_property(compilation_config, 'vm_type', class: String, optional: true)
-
         if vm_type_name
           @vm_type = vm_types.find { |vm_type| vm_type.name == vm_type_name }
 
           if @vm_type.nil?
             vm_types_names = vm_types.map { |vm_type| vm_type.name }
             raise Bosh::Director::CompilationConfigInvalidVmType,
-              "Compilation config references unknown vm type '#{vm_type_name}'. Known vm types are: #{vm_types_names.join(', ')}"
+                  "Compilation config references unknown vm type '#{vm_type_name}'. Known vm types are: #{vm_types_names.join(', ')}"
           end
 
           unless @cloud_properties.empty?
             raise Bosh::Director::CompilationConfigCloudPropertiesNotAllowed,
-              "Compilation config is using vm type '#{@vm_type.name}' and should not be configuring cloud_properties."
+                  "Compilation config is using vm type '#{@vm_type.name}' and should not be configuring cloud_properties."
           end
         end
       end
 
-      def availability_zone_name
-        @availability_zone.nil? ? nil : @availability_zone.name
+      def parse_vm_extensions(compilation_config, vm_extensions)
+        @vm_extensions = []
+
+        vm_extension_names = safe_property(compilation_config, 'vm_extensions', class: Array, optional: true)
+        Array(vm_extension_names).each { |vm_extension_name|
+          vm_extension = vm_extensions.find { |vm_extension| vm_extension.name == vm_extension_name }
+
+          if vm_extension.nil?
+            vm_extensions_names = vm_extensions.map { |vm_extension| vm_extension.name }
+            raise Bosh::Director::CompilationConfigInvalidVmExtension,
+                  "Compilation config references unknown vm extension '#{vm_extension_name}'. Known vm extensions are: #{vm_extensions_names.join(', ')}"
+          end
+
+          if @vm_type.nil?
+            raise Bosh::Director::CompilationConfigVmTypeRequired,
+                  "Compilation config is using vm extension '#{vm_extension.name}' and must configure a vm type."
+          end
+
+          @vm_extensions.push(vm_extension)
+        }
       end
     end
   end

data/lib/bosh/director/deployment_plan/compilation_instance_pool.rb

@@ -115,9 +115,11 @@ module Bosh::Director
           vm_type = CompilationVmType.new(@deployment_plan.compilation.cloud_properties)
         end
 
+        vm_extensions = @deployment_plan.compilation.vm_extensions
+
         env = Env.new(@deployment_plan.compilation.env)
 
-        compile_job = CompilationJob.new(vm_type, stemcell, env, @deployment_plan.compilation.network_name)
+        compile_job = CompilationJob.new(vm_type, vm_extensions, stemcell, env, @deployment_plan.compilation.network_name)
         availability_zone = @deployment_plan.compilation.availability_zone
         instance = Instance.create_from_job(compile_job, 0, 'started', @deployment_plan.model, {}, availability_zone, @logger)
         instance.bind_new_instance_model
@@ -174,11 +176,12 @@ module Bosh::Director
     end
 
     class CompilationJob
-      attr_reader :vm_type, :stemcell, :env, :name
+      attr_reader :vm_type, :vm_extensions, :stemcell, :env, :name
       attr_reader :instance_plans
 
-      def initialize(vm_type, stemcell, env, compilation_network_name)
+      def initialize(vm_type, vm_extensions, stemcell, env, compilation_network_name)
         @vm_type = vm_type
+        @vm_extensions = vm_extensions
         @stemcell = stemcell
         @env = env
         @network = compilation_network_name
@@ -219,6 +222,10 @@ module Bosh::Director
         {}
       end
 
+      def update_spec
+        nil
+      end
+
       def persistent_disk_type
         nil
       end

data/lib/bosh/director/deployment_plan/deployment_repo.rb

@@ -1,24 +1,18 @@
 module Bosh::Director
   module DeploymentPlan
     class DeploymentRepo
-      def initialize
-        @permission_authorizer = Bosh::Director::PermissionAuthorizer.new
-      end
 
       def find_or_create_by_name(name, options={})
         attributes = {name: name}
         deployment = Bosh::Director::Models::Deployment.find(attributes)
 
-        if options['scopes']
-          attributes.merge!(scopes: options['scopes'].join(','))
-        end
+        return deployment if deployment
 
-        if options['scopes'] && deployment
-          @permission_authorizer.raise_error_if_unauthorized(options['scopes'], deployment.scopes.split(','))
+        if options['scopes']
+          team_scopes = Bosh::Director::Models::Deployment.transform_admin_team_scope_to_teams(options['scopes'])
+          attributes.merge!(teams: team_scopes.join(','))
         end
 
-        return deployment if deployment
-
         create_for_attributes(attributes)
       end
 

data/lib/bosh/director/deployment_plan/dynamic_network.rb

@@ -124,7 +124,7 @@ module Bosh::Director
       def validate_reference_from_job!(job_network_spec, job_name)
         if job_network_spec.has_key?('static_ips')
           raise JobStaticIPNotSupportedOnDynamicNetwork,
-            "Job '#{job_name}' using dynamic network '#{name}' cannot specify static IP(s)"
+            "Instance group '#{job_name}' using dynamic network '#{name}' cannot specify static IP(s)"
         end
       end
 

data/lib/bosh/director/deployment_plan/instance.rb

@@ -25,8 +25,6 @@ module Bosh::Director
       # @return [String] job state
       attr_reader :virtual_state
 
-      attr_reader :current_state
-
       attr_reader :availability_zone
 
       attr_reader :existing_network_reservations
@@ -37,6 +35,7 @@ module Bosh::Director
           index,
           virtual_state,
           job.vm_type,
+          job.vm_extensions,
           job.stemcell,
           job.env,
           job.compilation?,
@@ -52,6 +51,7 @@ module Bosh::Director
         index,
         virtual_state,
         vm_type,
+        vm_extensions,
         stemcell,
         env,
         compilation,
@@ -66,6 +66,7 @@ module Bosh::Director
         @deployment_model = deployment_model
         @job_name = job_name
         @vm_type = vm_type
+        @vm_extensions = vm_extensions
         @stemcell = stemcell
         @env = env
         @compilation = compilation
@@ -73,6 +74,9 @@ module Bosh::Director
         @configuration_hash = nil
         @template_hashes = nil
         @vm = nil
+
+        # This state is coming from the agent, we
+        # only need networks and job_state from it.
         @current_state = instance_state || {}
 
         # reservation generated from current state/DB
@@ -124,6 +128,10 @@ module Bosh::Director
         @vm_type
       end
 
+      def vm_extensions
+        @vm_extensions
+      end
+
       def stemcell
         @stemcell
       end
@@ -198,31 +206,30 @@ module Bosh::Director
         changed
       end
 
-      ##
-      # @return [Boolean] returns true if the expected configuration hash
-      #   differs from the one provided by the VM
-      def configuration_changed?
-        changed = configuration_hash != @current_state['configuration_hash']
-        log_changes(__method__, @current_state['configuration_hash'], configuration_hash) if changed
-        changed
-      end
-
       def current_job_spec
-        @current_state['job']
+        @model.spec['job']
       end
 
       def current_packages
-        @current_state['packages']
+        @model.spec['packages']
       end
 
       def current_job_state
         @current_state['job_state']
       end
 
+      def current_networks
+        @current_state['networks']
+      end
+
       def update_state
         @model.update(state: state)
       end
 
+      def dirty?
+        !@model.update_completed
+      end
+
       def update_description
         @model.update(job: job_name, index: index)
       end
@@ -269,11 +276,19 @@ module Bosh::Director
       end
 
       def cloud_properties
-        if @availability_zone.nil?
-          vm_type.cloud_properties
-        else
-          @availability_zone.cloud_properties.merge(vm_type.cloud_properties)
+        merged_cloud_properties = nil
+
+        if !@availability_zone.nil?
+          merged_cloud_properties = merge_cloud_properties(merged_cloud_properties, @availability_zone.cloud_properties)
+        end
+
+        merged_cloud_properties = merge_cloud_properties(merged_cloud_properties, vm_type.cloud_properties)
+
+        Array(vm_extensions).each do |vm_extension|
+          merged_cloud_properties = merge_cloud_properties(merged_cloud_properties, vm_extension.cloud_properties)
         end
+
+        merged_cloud_properties
       end
 
       def availability_zone_name
@@ -294,6 +309,10 @@ module Bosh::Director
 
       private
 
+      def merge_cloud_properties(merged_cloud_properties, new_cloud_properties)
+        merged_cloud_properties.nil? ? new_cloud_properties : merged_cloud_properties.merge(new_cloud_properties)
+      end
+
       # Looks up instance model in DB
       # @return [Models::Instance]
       def find_or_create_model