bosh_vsphere_cpi 2.2.0 → 2.2.1

data/lib/cloud/vsphere/resources.rb

@@ -1,154 +1,5 @@
-require 'cloud/vsphere/resources/datacenter'
-
 module VSphereCloud
-  class Resources
-    include ObjectStringifier
-    stringify_with :datacenter
-    MEMORY_HEADROOM = 128
-    DISK_HEADROOM = 1024
-    STALE_TIMEOUT = 60
+  module Resources
     BYTES_IN_MB = 1024 * 1024
-
-    attr_reader :drs_rules
-
-    def initialize(datacenter, config)
-      @datacenter = datacenter
-      @config = config
-      @logger = config.logger
-      @last_update = 0
-      @lock = Monitor.new
-      @drs_rules = []
-    end
-
-    # Place the persistent datastore in the given datacenter and cluster with
-    # the requested disk space.
-    #
-    # @param [String] cluster_name cluster name.
-    # @param [Integer] disk_size_in_mb disk size in mb.
-    # @return [Datastore?] datastore if it was placed succesfuly.
-    def pick_persistent_datastore_in_cluster(cluster_name, disk_size_in_mb)
-      @lock.synchronize do
-        cluster = @datacenter.clusters[cluster_name]
-        if cluster.nil?
-          raise Bosh::Clouds::CloudError, "Couldn't find cluster '#{cluster_name}'. Found: #{@datacenter.clusters.values.map(&:name)}"
-        end
-
-        datastore = cluster.pick_persistent(disk_size_in_mb)
-        datastore.allocate(disk_size_in_mb)
-        datastore
-      end
-    end
-
-    # Find a cluster for a vm with the requested memory and ephemeral storage, attempting
-    # to allocate it near existing persistent disks.
-    #
-    # @param [Integer] requested_memory_in_mb requested memory.
-    # @param [Integer] requested_ephemeral_disk_size_in_mb requested ephemeral storage.
-    # @param [Array<Resources::Disk>] existing_persistent_disks existing persistent disks, if any.
-    # @return [Cluster] selected cluster if the resources were placed successfully, otherwise raises.
-    def pick_cluster_for_vm(requested_memory_in_mb, requested_ephemeral_disk_size_in_mb, existing_persistent_disks)
-      @lock.synchronize do
-        # calculate locality to prioritizing clusters that contain the most persistent data.
-        clusters = @datacenter.clusters.values
-        persistent_disk_index = PersistentDiskIndex.new(clusters, existing_persistent_disks)
-
-        scored_clusters = clusters.map do |cluster|
-          persistent_disk_not_in_this_cluster = existing_persistent_disks.reject do |disk|
-            persistent_disk_index.clusters_connected_to_disk(disk).include?(cluster)
-          end
-
-          score = Scorer.score(
-            @config.logger,
-            cluster,
-            requested_memory_in_mb,
-            requested_ephemeral_disk_size_in_mb,
-            persistent_disk_not_in_this_cluster.map(&:size_in_mb)
-          )
-
-          [cluster, score]
-        end
-
-        acceptable_clusters = scored_clusters.select { |_, score| score > 0 }
-
-        @logger.debug("Acceptable clusters: #{acceptable_clusters.inspect}")
-
-        if acceptable_clusters.empty?
-          total_persistent_size = existing_persistent_disks.map(&:size_in_mb).inject(0, :+)
-          cluster_infos = clusters.map { |cluster| describe_cluster(cluster) }
-
-          raise "Unable to allocate vm with #{requested_memory_in_mb}mb RAM, " +
-              "#{requested_ephemeral_disk_size_in_mb / 1024}gb ephemeral disk, " +
-              "and #{total_persistent_size / 1024}gb persistent disk from any cluster.\n#{cluster_infos.join(", ")}."
-        end
-
-        acceptable_clusters = acceptable_clusters.sort_by do |cluster, _score|
-          persistent_disk_index.disks_connected_to_cluster(cluster).map(&:size_in_mb).inject(0, :+)
-        end.reverse
-
-        if acceptable_clusters.any? { |cluster, _| persistent_disk_index.disks_connected_to_cluster(cluster).any? }
-          @logger.debug('Choosing cluster with the greatest available disk')
-          selected_cluster, _ = acceptable_clusters.first
-        else
-          @logger.debug('Choosing cluster by weighted random')
-          selected_cluster = Util.weighted_random(acceptable_clusters)
-        end
-
-        @logger.debug("Selected cluster '#{selected_cluster.name}'")
-
-        selected_cluster.allocate(requested_memory_in_mb)
-        selected_cluster
-      end
-    end
-
-    def describe_cluster(cluster)
-      "#{cluster.name} has #{cluster.free_memory}mb/" +
-        "#{cluster.total_free_ephemeral_disk_in_mb / 1024}gb/" +
-        "#{cluster.total_free_persistent_disk_in_mb / 1024}gb"
-    end
-
-    def pick_ephemeral_datastore(cluster, disk_size_in_mb)
-      @lock.synchronize do
-        datastore = cluster.pick_ephemeral(disk_size_in_mb)
-        datastore.allocate(disk_size_in_mb)
-        datastore
-      end
-    end
-
-    def pick_persistent_datastore(cluster, disk_size_in_mb)
-      @lock.synchronize do
-        datastore = cluster.pick_persistent(disk_size_in_mb)
-        datastore.allocate(disk_size_in_mb)
-        datastore
-      end
-    end
-
-    private
-
-    attr_reader :config, :datacenter
-
-
-    class PersistentDiskIndex
-      def initialize(clusters, existing_persistent_disks)
-        @clusters_to_disks = Hash[*clusters.map do |cluster|
-            [cluster, existing_persistent_disks.select { |disk| cluster_includes_datastore?(cluster, disk.datastore) }]
-          end.flatten(1)]
-
-        @disks_to_clusters = Hash[*existing_persistent_disks.map do |disk|
-            [disk, clusters.select { |cluster| cluster_includes_datastore?(cluster, disk.datastore) }]
-          end.flatten(1)]
-      end
-
-      def cluster_includes_datastore?(cluster, datastore)
-        cluster.persistent(datastore.name) != nil
-      end
-
-      def disks_connected_to_cluster(cluster)
-        @clusters_to_disks[cluster]
-      end
-
-      def clusters_connected_to_disk(disk)
-        @disks_to_clusters[disk]
-      end
-    end
   end
 end

data/lib/cloud/vsphere/vm_creator.rb

@@ -1,7 +1,7 @@
 module VSphereCloud
   class VmCreator
-    def initialize(memory, disk_size_in_mb, cpu, nested_hardware_virtualization, placer, client, cloud_searcher, logger, cpi, agent_env, file_provider, disk_provider)
-      @placer = placer
+    def initialize(memory, disk_size_in_mb, cpu, nested_hardware_virtualization, drs_rules, client, cloud_searcher, logger, cpi, agent_env, file_provider, datacenter, cluster=nil)
+      @drs_rules = drs_rules
       @client = client
       @cloud_searcher = cloud_searcher
       @logger = logger
@@ -12,9 +12,10 @@ module VSphereCloud
       @nested_hardware_virtualization = nested_hardware_virtualization
       @agent_env = agent_env
       @file_provider = file_provider
-      @disk_provider = disk_provider
+      @datacenter = datacenter
+      @cluster = cluster
 
-      @logger.debug("VM creator initialized with memory: #{@memory}, disk: #{@disk}, cpu: #{@cpu}, placer: #{@placer}")
+      @logger.debug("VM creator initialized with memory: #{@memory}, disk: #{@disk}, cpu: #{@cpu}")
     end
 
     def create(agent_id, stemcell_cid, networks, persistent_disk_cids, environment)
@@ -31,12 +32,15 @@ module VSphereCloud
       stemcell_size /= 1024 * 1024
 
       persistent_disk_cids ||= []
-      persistent_disks = persistent_disk_cids.map { |cid| @disk_provider.find(cid) }
+      persistent_disks = persistent_disk_cids.map { |cid| @datacenter.find_disk(cid) }
 
       # need to include swap and linked clone log
       ephemeral_disk_size_in_mb = @disk_size_in_mb + @memory + stemcell_size
-      cluster = @placer.pick_cluster_for_vm(@memory, ephemeral_disk_size_in_mb, persistent_disks)
-      datastore = @placer.pick_ephemeral_datastore(cluster, ephemeral_disk_size_in_mb)
+      cluster = @cluster || @datacenter.pick_cluster_for_vm(@memory, ephemeral_disk_size_in_mb, persistent_disks)
+      if cluster.ephemeral_datastores.empty?
+        raise "Cluster '#{cluster.name}' has no ephemeral datastores"
+      end
+      datastore = @datacenter.pick_ephemeral_datastore(ephemeral_disk_size_in_mb, cluster.ephemeral_datastores.keys)
 
       vm_cid = "vm-#{SecureRandom.uuid}"
       @logger.info("Creating vm: #{vm_cid} on #{cluster.mob} stored in #{datastore.mob}")
@@ -54,20 +58,22 @@ module VSphereCloud
       config = VimSdk::Vim::Vm::ConfigSpec.new(config_hash)
       config.device_change = []
 
-      ephemeral_disk = VSphereCloud::EphemeralDisk.new(@disk_size_in_mb, vm_cid, datastore)
-      ephemeral_disk_config = ephemeral_disk.create_spec(replicated_stemcell_vm.system_disk.controller_key)
+      ephemeral_disk = Resources::EphemeralDisk.new(Resources::EphemeralDisk::DISK_NAME, @disk_size_in_mb, datastore, vm_cid)
+
+      ephemeral_disk_config = ephemeral_disk.create_disk_attachment_spec(replicated_stemcell_vm.system_disk.controller_key)
       config.device_change << ephemeral_disk_config
 
       dvs_index = {}
       networks.each_value do |network|
         v_network_name = network['cloud_properties']['name']
         network_mob = @client.find_by_inventory_path([cluster.datacenter.name, 'network', v_network_name])
-        nic_config = @cpi.create_nic_config_spec(v_network_name, network_mob, replicated_stemcell_vm.pci_controller.key, dvs_index)
+        virtual_nic = Resources::Nic.create_virtual_nic(@cloud_searcher, v_network_name, network_mob, replicated_stemcell_vm.pci_controller.key, dvs_index)
+        nic_config = Resources::VM.create_add_device_spec(virtual_nic)
         config.device_change << nic_config
       end
 
       replicated_stemcell_vm.nics.each do |nic|
-        nic_config = @cpi.create_delete_device_spec(nic)
+        nic_config = Resources::VM.create_delete_device_spec(nic)
         config.device_change << nic_config
       end
 
@@ -117,15 +123,16 @@ module VSphereCloud
       vm_cid
     end
 
+    private
+
     def create_drs_rules(vm, cluster)
-      return unless @placer.drs_rules
-      return if @placer.drs_rules.size == 0
+      return if @drs_rules.nil? || @drs_rules.size == 0
 
-      if @placer.drs_rules.size > 1
+      if @drs_rules.size > 1
         raise 'vSphere CPI supports only one DRS rule per resource pool'
       end
 
-      rule_config = @placer.drs_rules.first
+      rule_config = @drs_rules.first
 
       if rule_config['type'] != 'separate_vms'
         raise "vSphere CPI only supports DRS rule of 'separate_vms' type, not '#{rule_config['type']}'"

data/lib/cloud/vsphere/vm_creator_builder.rb

@@ -2,20 +2,21 @@ require 'cloud/vsphere/vm_creator'
 
 module VSphereCloud
   class VmCreatorBuilder
-    def build(resources, cloud_properties, client, cloud_searcher, logger, cpi, agent_env, file_provider, disk_provider)
+    def build(cloud_properties, client, cloud_searcher, logger, cpi, agent_env, file_provider, datacenter, cluster=nil, drs_rules=[])
       VmCreator.new(
         cloud_properties.fetch('ram'),
         cloud_properties.fetch('disk'),
         cloud_properties.fetch('cpu'),
         cloud_properties.fetch('nested_hardware_virtualization', false),
-        resources,
+        drs_rules,
         client,
         cloud_searcher,
         logger,
         cpi,
         agent_env,
         file_provider,
-        disk_provider
+        datacenter,
+        cluster
       )
     end
   end

data/lib/cloud/vsphere.rb

@@ -12,9 +12,7 @@ require 'cloud/vsphere/cloud'
 require 'cloud/vsphere/cloud_searcher'
 require 'cloud/vsphere/config'
 require 'cloud/vsphere/cluster_config'
-require 'cloud/vsphere/disk_provider'
 require 'cloud/vsphere/file_provider'
-require 'cloud/vsphere/fixed_cluster_placer'
 require 'cloud/vsphere/lease_obtainer'
 require 'cloud/vsphere/lease_updater'
 require 'cloud/vsphere/path_finder'
@@ -22,12 +20,18 @@ require 'cloud/vsphere/resources/cluster_provider'
 require 'cloud/vsphere/resources/cluster'
 require 'cloud/vsphere/resources/datacenter'
 require 'cloud/vsphere/resources/datastore'
-require 'cloud/vsphere/resources/disk/ephemeral_disk'
+require 'cloud/vsphere/resources/disk'
+require 'cloud/vsphere/resources/ephemeral_disk'
+require 'cloud/vsphere/resources/persistent_disk'
+require 'cloud/vsphere/resources/nic'
 require 'cloud/vsphere/resources/folder'
 require 'cloud/vsphere/resources/vm'
 require 'cloud/vsphere/resources/resource_pool'
 require 'cloud/vsphere/resources/scorer'
 require 'cloud/vsphere/resources/util'
+require 'cloud/vsphere/drs_rules/drs_lock'
+require 'cloud/vsphere/drs_rules/drs_rule'
+require 'cloud/vsphere/drs_rules/vm_attribute_manager'
 require 'cloud/vsphere/soap_stub'
 require 'cloud/vsphere/vm_creator_builder'
 require 'cloud/vsphere/vm_provider'
@@ -46,7 +50,7 @@ module Bosh
                      :create_disk, :has_disk?, :delete_disk,
                      :attach_disk, :detach_disk,
                      :snapshot_disk, :delete_snapshot,
-                     :current_vm_id, :get_disks, :ping, :disk_provider
+                     :current_vm_id, :get_disks, :ping
 
       def initialize(options)
         @delegate = VSphereCloud::Cloud.new(options)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh_vsphere_cpi
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.2.1
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-17 00:00:00.000000000 Z
+date: 2016-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_common
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.7.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.4.0
+        version: 2.7.1
 - !ruby/object:Gem::Dependency
   name: mono_logger
   requirement: !ruby/object:Gem::Requirement
@@ -125,12 +125,10 @@ files:
 - lib/cloud/vsphere/cloud_searcher.rb
 - lib/cloud/vsphere/cluster_config.rb
 - lib/cloud/vsphere/config.rb
-- lib/cloud/vsphere/disk_provider.rb
 - lib/cloud/vsphere/drs_rules/drs_lock.rb
 - lib/cloud/vsphere/drs_rules/drs_rule.rb
 - lib/cloud/vsphere/drs_rules/vm_attribute_manager.rb
 - lib/cloud/vsphere/file_provider.rb
-- lib/cloud/vsphere/fixed_cluster_placer.rb
 - lib/cloud/vsphere/ip_conflict_detector.rb
 - lib/cloud/vsphere/lease_obtainer.rb
 - lib/cloud/vsphere/lease_updater.rb
@@ -142,9 +140,10 @@ files:
 - lib/cloud/vsphere/resources/datacenter.rb
 - lib/cloud/vsphere/resources/datastore.rb
 - lib/cloud/vsphere/resources/disk.rb
-- lib/cloud/vsphere/resources/disk/disk_config.rb
-- lib/cloud/vsphere/resources/disk/ephemeral_disk.rb
+- lib/cloud/vsphere/resources/ephemeral_disk.rb
 - lib/cloud/vsphere/resources/folder.rb
+- lib/cloud/vsphere/resources/nic.rb
+- lib/cloud/vsphere/resources/persistent_disk.rb
 - lib/cloud/vsphere/resources/resource_pool.rb
 - lib/cloud/vsphere/resources/scorer.rb
 - lib/cloud/vsphere/resources/util.rb

data/lib/cloud/vsphere/disk_provider.rb

@@ -1,94 +0,0 @@
-require 'ostruct'
-require 'cloud/vsphere/resources/disk'
-
-module VSphereCloud
-  class DiskProvider
-  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvim.VirtualDiskManager.VirtualDiskType.html
-  SUPPORTED_DISK_TYPES = %w{
-      eagerZeroedThick
-      preallocated
-      thick
-      thin
-    }
-
-    DEFAULT_DISK_TYPE = 'preallocated'
-
-    def initialize(virtual_disk_manager, datacenter, resources, disk_path, client, logger)
-      @virtual_disk_manager = virtual_disk_manager
-      @datacenter = datacenter
-      @resources = resources
-      @disk_path = disk_path
-      @client = client
-      @logger = logger
-    end
-
-    def create(disk_size_in_mb, disk_type, cluster)
-      type = disk_type
-      if type.nil?
-        type = DEFAULT_DISK_TYPE
-      end
-
-      unless SUPPORTED_DISK_TYPES.include?(type)
-        raise "Disk type: '#{disk_type}' is not supported"
-      end
-
-      if cluster
-        datastore = @resources.pick_persistent_datastore_in_cluster(cluster.name, disk_size_in_mb)
-      else
-        datastore = @datacenter.pick_persistent_datastore(disk_size_in_mb)
-      end
-      disk_cid = "disk-#{SecureRandom.uuid}"
-      @logger.debug("Creating disk '#{disk_cid}' in datastore '#{datastore.name}'")
-
-      @client.create_disk(@datacenter, datastore, disk_cid, @disk_path, disk_size_in_mb, type)
-    end
-
-    def find_and_move(disk_cid, cluster, datacenter, accessible_datastores)
-      disk = find(disk_cid)
-      disk_in_persistent_datastore = @datacenter.persistent_datastores.include?(disk.datastore.name)
-      disk_in_accessible_datastore = accessible_datastores.include?(disk.datastore.name)
-      if disk_in_persistent_datastore && disk_in_accessible_datastore
-        @logger.info("Disk #{disk_cid} found in an accessible, persistent datastore '#{disk.datastore.name}'")
-        return disk
-      end
-
-      destination_datastore = @resources.pick_persistent_datastore_in_cluster(cluster.name, disk.size_in_mb)
-
-      unless accessible_datastores.include?(destination_datastore.name)
-        raise "Datastore '#{destination_datastore.name}' is not accessible to cluster '#{cluster.name}'"
-      end
-
-      destination_path = path(destination_datastore, disk_cid)
-      @logger.info("Moving #{disk.path} to #{destination_path}")
-      @client.move_disk(datacenter, disk.path, datacenter, destination_path)
-      @logger.info('Moved disk successfully')
-      Resources::Disk.new(disk_cid, disk.size_in_mb, destination_datastore, destination_path)
-    end
-
-    def find(disk_cid)
-      persistent_datastores = @datacenter.persistent_datastores
-      @logger.debug("Looking for disk #{disk_cid} in datastores: #{persistent_datastores}")
-      persistent_datastores.each do |_, datastore|
-        disk = @client.find_disk(disk_cid, datastore, @disk_path)
-        @logger.debug("disk #{disk_cid} found in: #{datastore}") unless disk.nil?
-        return disk unless disk.nil?
-      end
-
-      other_datastores = @datacenter.all_datastores.reject{|datastore_name, _| persistent_datastores[datastore_name] }
-      @logger.debug("disk #{disk_cid} not found in filtered persistent datastores, trying other datastores: #{other_datastores}")
-      other_datastores.each do |_, datastore|
-        disk = @client.find_disk(disk_cid, datastore, @disk_path)
-        @logger.debug("disk #{disk_cid} found in: #{datastore}") unless disk.nil?
-        return disk unless disk.nil?
-      end
-
-      raise Bosh::Clouds::DiskNotFound.new(false), "Could not find disk with id '#{disk_cid}'"
-    end
-
-    private
-
-    def path(datastore, disk_cid)
-      "[#{datastore.name}] #{@disk_path}/#{disk_cid}.vmdk"
-    end
-  end
-end

data/lib/cloud/vsphere/fixed_cluster_placer.rb

@@ -1,32 +0,0 @@
-module VSphereCloud
-  class FixedClusterPlacer
-    include ObjectStringifier
-    stringify_with :cluster
-
-    attr_reader :drs_rules
-
-    def initialize(cluster, drs_rules)
-      @cluster = cluster
-      @drs_rules = drs_rules
-    end
-
-    def pick_cluster_for_vm(memory, ephemeral, persistent)
-      @cluster.allocate(memory)
-      @cluster
-    end
-
-    def pick_ephemeral_datastore(cluster, disk_size_in_mb)
-      datastore = cluster.pick_ephemeral(disk_size_in_mb)
-      datastore.allocate(disk_size_in_mb)
-      datastore
-    end
-
-    def pick_persistent_datastore(_, _)
-      raise NotImplementedError
-    end
-
-    private
-
-    attr_reader :cluster
-  end
-end

data/lib/cloud/vsphere/resources/disk/disk_config.rb

@@ -1,35 +0,0 @@
-module VSphereCloud
-  class DiskConfig
-    def initialize(datastore_name, filename, controller_key, size_in_mb)
-      @datastore_name = datastore_name
-      @filename = filename
-      @controller_key = controller_key
-      @size_in_mb = size_in_mb
-    end
-
-    def spec(options)
-      backing_info = VimSdk::Vim::Vm::Device::VirtualDisk::FlatVer2BackingInfo.new
-      backing_info.datastore = @datastore_name
-      if options[:independent]
-        backing_info.disk_mode = VimSdk::Vim::Vm::Device::VirtualDiskOption::DiskMode::INDEPENDENT_PERSISTENT
-      else
-        backing_info.disk_mode = VimSdk::Vim::Vm::Device::VirtualDiskOption::DiskMode::PERSISTENT
-      end
-      backing_info.file_name = @filename
-
-      virtual_disk = VimSdk::Vim::Vm::Device::VirtualDisk.new
-      virtual_disk.key = -1
-      virtual_disk.controller_key = @controller_key
-      virtual_disk.backing = backing_info
-      virtual_disk.capacity_in_kb = @size_in_mb * 1024
-
-      device_config_spec = VimSdk::Vim::Vm::Device::VirtualDeviceSpec.new
-      device_config_spec.device = virtual_disk
-      device_config_spec.operation = VimSdk::Vim::Vm::Device::VirtualDeviceSpec::Operation::ADD
-      if options[:create]
-        device_config_spec.file_operation = VimSdk::Vim::Vm::Device::VirtualDeviceSpec::FileOperation::CREATE
-      end
-      device_config_spec
-    end
-  end
-end

data/lib/cloud/vsphere/resources/disk/ephemeral_disk.rb

@@ -1,22 +0,0 @@
-require 'cloud/vsphere/resources/disk/disk_config'
-
-module VSphereCloud
-  class EphemeralDisk
-    DISK_NAME = 'ephemeral_disk'
-    def initialize(size_in_mb, folder_name, datastore)
-      @folder_name = folder_name
-      @datastore = datastore
-      @size_in_mb = size_in_mb
-    end
-
-    def create_spec(controller_key)
-      DiskConfig.new(@datastore.mob, filename, controller_key, @size_in_mb).spec(create: true)
-    end
-
-    private
-
-    def filename
-      "[#{@datastore.name}] #{@folder_name}/#{DISK_NAME}.vmdk"
-    end
-  end
-end