bosh-director 1.3160.0 → 1.3163.0

data/lib/bosh/director/deployment_plan/availability_zone.rb

@@ -0,0 +1,27 @@
+module Bosh::Director
+  module DeploymentPlan
+    class AvailabilityZone
+      extend ValidationHelper
+
+      def self.parse(availability_zone_spec)
+        name = safe_property(availability_zone_spec, "name", class: String)
+
+        cloud_properties =
+          safe_property(availability_zone_spec, "cloud_properties", class: Hash, default: {})
+
+        new(name, cloud_properties)
+      end
+
+      def initialize(name, cloud_properties)
+        @name = name
+        @cloud_properties = cloud_properties
+      end
+
+      attr_reader :name, :cloud_properties
+
+      def inspect
+        "az: #{name}"
+      end
+    end
+  end
+end

data/lib/bosh/director/deployment_plan/cloud_manifest_parser.rb

@@ -3,72 +3,181 @@ module Bosh::Director
     class CloudManifestParser
       include ValidationHelper
 
-      def initialize(deployment, logger)
-        @deployment = deployment
+      def initialize(logger)
         @logger = logger
       end
 
-      def parse(cloud_manifest)
-        @cloud_manifest = cloud_manifest
+      def parse(cloud_manifest, global_network_resolver, ip_provider_factory)
+        azs = parse_availability_zones(cloud_manifest)
+        az_list = CloudPlanner.index_by_name(azs)
+        networks = parse_networks(cloud_manifest, global_network_resolver, azs)
+        compilation_config = parse_compilation(cloud_manifest, networks, az_list)
+        resource_pools = parse_resource_pools(cloud_manifest)
+        vm_types = parse_vm_types(cloud_manifest)
+        disk_types = parse_disk_types(cloud_manifest)
 
-        parse_networks
-        parse_compilation
-        parse_resource_pools
-        parse_disk_pools
-
-        @deployment
+        CloudPlanner.new({
+          availability_zones_list: az_list,
+          networks: networks,
+          global_network_resolver: global_network_resolver,
+          ip_provider_factory: ip_provider_factory,
+          compilation: compilation_config,
+          resource_pools: resource_pools,
+          vm_types: vm_types,
+          disk_types: disk_types,
+          logger: @logger,
+        })
       end
 
       private
 
-      def parse_networks
-        networks = safe_property(@cloud_manifest, 'networks', :class => Array)
-        networks.each do |network_spec|
-          type = safe_property(network_spec, 'type', :class => String,
-            :default => 'manual')
+      def parse_availability_zones(cloud_manifest)
+        availability_zones = safe_property(cloud_manifest, 'azs', :class => Array, :optional => true, :default => [])
+        parsed_availability_zones = availability_zones.map do |availability_zone|
+          AvailabilityZone.parse(availability_zone)
+        end
+
+        duplicates = detect_duplicates(parsed_availability_zones) { |az| az.name }
+        unless duplicates.empty?
+          raise DeploymentDuplicateAvailabilityZoneName, "Duplicate az name `#{duplicates.first.name}'"
+        end
+
+        parsed_availability_zones
+      end
+
+      def parse_networks(cloud_manifest, global_network_resolver, availability_zones)
+        networks = safe_property(cloud_manifest, 'networks', :class => Array)
+        if networks.empty?
+          raise DeploymentNoNetworks, 'No networks specified'
+        end
+
+        parsed_networks = networks.map do |network_spec|
+          type = safe_property(network_spec, 'type', :class => String, :default => 'manual')
 
           case type
             when 'manual'
-              network = ManualNetwork.new(@deployment, network_spec)
+              ManualNetwork.parse(network_spec, availability_zones, global_network_resolver, @logger)
             when 'dynamic'
-              network = DynamicNetwork.new(@deployment, network_spec)
+              DynamicNetwork.parse(network_spec, availability_zones, @logger)
             when 'vip'
-              network = VipNetwork.new(@deployment, network_spec)
+              VipNetwork.new(network_spec, @logger)
             else
               raise DeploymentInvalidNetworkType,
                 "Invalid network type `#{type}'"
           end
-
-          @deployment.add_network(network)
         end
 
-        if @deployment.networks.empty?
-          raise DeploymentNoNetworks, 'No networks specified'
+        duplicates = detect_duplicates(parsed_networks) { |network| network.canonical_name }
+        unless duplicates.empty?
+          raise DeploymentCanonicalNetworkNameTaken, "Invalid network name `#{duplicates.first.name}', canonical name already taken"
         end
+
+        parsed_networks
       end
 
-      def parse_compilation
-        compilation_spec = safe_property(@cloud_manifest, 'compilation', :class => Hash)
-        @deployment.compilation = CompilationConfig.new(@deployment, compilation_spec)
+      def parse_compilation(cloud_manifest, networks, az_list)
+        compilation_spec = safe_property(cloud_manifest, 'compilation', :class => Hash)
+        config = CompilationConfig.new(compilation_spec, az_list)
+
+        compilation_network = networks.find { |network| network.name == config.network_name }
+        if compilation_network.nil?
+          raise CompilationConfigUnknownNetwork,
+            "Compilation config references an unknown " +
+              "network `#{config.network_name}'"
+        end
+
+        unless compilation_network.has_azs?([config.availability_zone_name])
+          raise JobNetworkMissingRequiredAvailabilityZone,
+            "Compilation config refers to az '#{config.availability_zone_name}' but network '#{compilation_network.name}' has no matching subnet(s)."
+        end
+
+        config
       end
 
-      def parse_resource_pools
-        resource_pools = safe_property(@cloud_manifest, 'resource_pools', :class => Array)
-        resource_pools.each do |rp_spec|
-          @deployment.add_resource_pool(ResourcePool.new(@deployment, rp_spec, @logger))
+      def parse_resource_pools(cloud_manifest)
+        resource_pools = safe_property(cloud_manifest, 'resource_pools', :class => Array, optional: true)
+
+        if resource_pools.nil?
+          return {}
         end
 
-        if @deployment.resource_pools.empty?
+        if resource_pools.empty?
           raise DeploymentNoResourcePools, 'No resource_pools specified'
         end
+
+        parsed_resource_pools = resource_pools.map do |rp_spec|
+          ResourcePool.new(rp_spec)
+        end
+
+        duplicates = detect_duplicates(parsed_resource_pools) { |rp| rp.name }
+        unless duplicates.empty?
+          raise DeploymentDuplicateResourcePoolName, "Duplicate resource pool name `#{duplicates.first.name}'"
+        end
+
+        parsed_resource_pools
+      end
+
+      def parse_vm_types(cloud_manifest)
+        vm_types = safe_property(cloud_manifest, 'vm_types', :class => Array, :optional => true, :default => [])
+
+        parsed_vm_types = vm_types.map do |vmt_spec|
+          VmType.new(vmt_spec)
+        end
+
+        duplicates = detect_duplicates(parsed_vm_types) { |vmt| vmt.name }
+        unless duplicates.empty?
+          raise DeploymentDuplicateVmTypeName, "Duplicate vm type name `#{duplicates.first.name}'"
+        end
+
+        parsed_vm_types
+      end
+
+
+      def parse_disk_types(cloud_manifest)
+        disk_pools = safe_property(cloud_manifest, 'disk_pools', :class => Array, :optional => true, :default => [])
+        disk_types = safe_property(cloud_manifest, 'disk_types', :class => Array, :optional => true, :default => [])
+
+        if disk_pools.any? && disk_types.any?
+          raise DeploymentInvalidDiskSpecification, "Both 'disk_types' and 'disk_pools' are specified, only one key is allowed. 'disk_pools' key will be DEPRECATED in the future."
+        end
+
+        disk_names = []
+
+        if disk_pools.any?
+          disk_source = 'pool'
+          disk_names = disk_pools
+        elsif disk_types.any?
+          disk_source = 'type'
+          disk_names = disk_types
+        end
+
+        parsed_disk_types = disk_names.map do |dp_spec|
+          DiskType.parse(dp_spec)
+        end
+
+
+        duplicates = detect_duplicates(parsed_disk_types) { |dp| dp.name }
+        unless duplicates.empty?
+          raise DeploymentDuplicateDiskTypeName, "Duplicate disk #{disk_source} name `#{duplicates.first.name}'"
+        end
+
+        parsed_disk_types
       end
 
-      def parse_disk_pools
-        disk_pools = safe_property(@cloud_manifest, 'disk_pools', :class => Array, :optional => true)
-        return if disk_pools.nil?
-        disk_pools.each do |dp_spec|
-          @deployment.add_disk_pool(DiskPool.parse(dp_spec))
+      def detect_duplicates(collection, &iteratee)
+        transformed_elements = Set.new
+        duplicated_elements = Set.new
+        collection.each do |element|
+          transformed = iteratee.call(element)
+
+          if transformed_elements.include?(transformed)
+            duplicated_elements << element
+          else
+            transformed_elements << transformed
+          end
         end
+
+        duplicated_elements
       end
     end
   end

data/lib/bosh/director/deployment_plan/compilation_config.rb

@@ -7,15 +7,9 @@ module Bosh::Director
     class CompilationConfig
       include ValidationHelper
 
-      # @return [DeploymentPlan] associated deployment
-      attr_accessor :deployment
-
       # @return [Integer] number of worker VMs to use
       attr_accessor :workers
 
-      # @return [DeploymentPlan::Network] network used by compilation workers
-      attr_accessor :network
-
       # @return [Hash] cloud properties to use when creating VMs. (optional)
       attr_accessor :cloud_properties
 
@@ -25,29 +19,37 @@ module Bosh::Director
       # @return [Bool] if VMs should be reused for compilation tasks. (optional)
       attr_accessor :reuse_compilation_vms
 
+      attr_reader :network_name
+
+      attr_reader :availability_zone
+
       # Creates compilation configuration spec from the deployment manifest.
       # @param [DeploymentPlan] deployment
       # @param [Hash] compilation_config parsed compilation config YAML section
-      def initialize(deployment, compilation_config)
-        @deployment = deployment
-        @workers = safe_property(compilation_config, "workers", class: Integer, min: 1)
+      def initialize(compilation_config, azs_list = {})
+        @workers = safe_property(compilation_config, 'workers', class: Integer, min: 1)
 
-        network_name = safe_property(compilation_config, "network", class: String)
+        @network_name = safe_property(compilation_config, 'network', class: String)
 
         @reuse_compilation_vms = safe_property(compilation_config,
-          "reuse_compilation_vms",
+          'reuse_compilation_vms',
           class: :boolean,
           optional: true)
 
-        @network = deployment.network(network_name)
-        if @network.nil?
-          raise CompilationConfigUnknownNetwork,
-            "Compilation config references an unknown " +
-              "network `#{network_name}'"
-        end
         @cloud_properties = safe_property(
-          compilation_config, "cloud_properties", class: Hash, default: {})
-        @env = safe_property(compilation_config, "env", class: Hash, optional: true, default: {})
+          compilation_config, 'cloud_properties', class: Hash, default: {})
+        @env = safe_property(compilation_config, 'env', class: Hash, optional: true, default: {})
+
+        az_name = safe_property(compilation_config, 'az', class: String, optional: true)
+        @availability_zone = azs_list[az_name]
+        if az_name && !az_name.empty? && @availability_zone.nil?
+          raise Bosh::Director::CompilationConfigInvalidAvailabilityZone,
+            "Compilation config references unknown az '#{az_name}'. Known azs are: [#{azs_list.keys.join(', ')}]"
+        end
+      end
+
+      def availability_zone_name
+        @availability_zone.nil? ? nil : @availability_zone.name
       end
     end
   end

data/lib/bosh/director/deployment_plan/compilation_instance_pool.rb

@@ -0,0 +1,169 @@
+module Bosh::Director
+  module DeploymentPlan
+    class CompilationInstancePool
+      def initialize(instance_reuser, vm_creator, deployment_plan, logger, instance_deleter)
+        @instance_reuser = instance_reuser
+        @vm_creator = vm_creator
+        @deployment_plan =  deployment_plan
+        @logger = logger
+        @instance_deleter = instance_deleter
+      end
+
+      def with_reused_vm(stemcell)
+        begin
+          instance = @instance_reuser.get_instance(stemcell)
+          if instance.nil?
+            instance_plan, instance = create_instance_plan(stemcell)
+            configure_instance_plan(instance_plan)
+            @instance_reuser.add_in_use_instance(instance_plan.instance, stemcell)
+          else
+            @logger.info("Reusing compilation VM `#{instance.vm.model.cid}' for stemcell `#{stemcell.model.desc}'")
+          end
+
+          yield instance
+
+          @instance_reuser.release_instance(instance)
+        rescue => e
+          unless instance.nil? || instance_plan.nil?
+            @instance_reuser.remove_instance(instance)
+            delete_instance(instance_plan)
+          end
+          raise e
+        end
+      end
+
+      def with_single_use_vm(stemcell)
+        begin
+          instance_plan, instance = create_instance_plan(stemcell)
+          configure_instance_plan(instance_plan)
+          yield instance
+        ensure
+          delete_instance(instance_plan) unless instance.nil?
+        end
+      end
+
+      def delete_instances(number_of_workers)
+        ThreadPool.new(:max_threads => number_of_workers).wrap do |pool|
+           @instance_reuser.each do |instance|
+            pool.process do
+              @instance_reuser.remove_instance(instance)
+              instance_plan = DeploymentPlan::InstancePlan.new(
+                existing_instance: instance.model,
+                instance: instance,
+                desired_instance: DeploymentPlan::DesiredInstance.new,
+                network_plans: []
+              )
+              delete_instance(instance_plan)
+            end
+          end
+        end
+      end
+
+      private
+
+      def delete_instance(instance_plan)
+        @instance_deleter.delete_instance_plan(instance_plan, EventLog::NullStage.new)
+      end
+
+      def create_instance_plan(stemcell)
+        vm_type = CompilationVmType.new(@deployment_plan.compilation.cloud_properties)
+        env = Env.new(@deployment_plan.compilation.env)
+
+        @compile_job = CompilationJob.new(vm_type, stemcell, env, @deployment_plan.compilation.network_name)
+        availability_zone = @deployment_plan.compilation.availability_zone
+        instance = Instance.create_from_job(@compile_job, 0, 'started', @deployment_plan.model, {}, availability_zone, @logger)
+        instance.bind_new_instance_model
+
+        compilation_network = @deployment_plan.network(@deployment_plan.compilation.network_name)
+        reservation = DesiredNetworkReservation.new_dynamic(instance, compilation_network)
+        desired_instance = DeploymentPlan::DesiredInstance.new(@compile_job, nil)
+        instance_plan = DeploymentPlan::InstancePlan.new(
+          existing_instance: instance.model,
+          instance: instance,
+          desired_instance: desired_instance,
+          network_plans: [DeploymentPlan::NetworkPlanner::Plan.new(reservation: reservation)]
+        )
+
+        @compile_job.add_instance_plans([instance_plan])
+
+        return instance_plan, instance
+      end
+
+      def configure_instance_plan(instance_plan)
+        instance_plan.instance.bind_unallocated_vm
+
+        @deployment_plan.ip_provider.reserve(instance_plan.network_plans.first.reservation)
+        @vm_creator.create_for_instance_plan(instance_plan, [])
+      end
+    end
+
+    private
+
+    class CompilationVmType
+      attr_reader :cloud_properties
+
+      def initialize(cloud_properties)
+        @cloud_properties = cloud_properties
+      end
+
+      def spec
+        {}
+      end
+    end
+
+    class CompilationJob
+      attr_reader :vm_type, :stemcell, :env, :name
+      attr_reader :instance_plans
+
+      def initialize(vm_type, stemcell, env, compilation_network_name)
+        @vm_type = vm_type
+        @stemcell = stemcell
+        @env = env
+        @network = compilation_network_name
+        @name = "compilation-#{SecureRandom.uuid}"
+        @instance_plans = []
+      end
+
+      def default_network
+        {
+          'dns' => @network,
+          'gateway' => @network
+        }
+      end
+
+      def availability_zones
+        nil
+      end
+
+      def add_instance_plans(instance_plans)
+        @instance_plans = instance_plans
+      end
+
+      def spec
+        {
+          'name' => @name
+        }
+      end
+
+      def package_spec
+        {}
+      end
+
+      def properties
+        {}
+      end
+
+      def link_spec
+        {}
+      end
+
+      def persistent_disk_type
+        nil
+      end
+
+      def compilation?
+        true
+      end
+    end
+  end
+end