bosh_vsphere_cpi 1.2865.0 → 1.2881.0

data/lib/cloud/vsphere/resources/scorer.rb

@@ -2,128 +2,71 @@
 
 module VSphereCloud
   class Resources
-
-    # Resource Scorer.
     class Scorer
 
-      # Creates a new Scorer given a cluster and requested memory and storage.
-      #
-      # @param [Cluster] cluster requested cluster.
-      # @param [Integer] memory required memory.
-      # @param [Array<Integer>] ephemeral list of required ephemeral disk sizes.
-      # @param [Array<Integer>] persistent list of required persistent disk
-      #   sizes.
-      def initialize(config, cluster, memory, ephemeral, persistent)
-        @logger = config.logger
-        @cluster = cluster
-        @memory = memory
-        @ephemeral = ephemeral
-        @persistent = persistent
-
-        @free_memory = cluster.free_memory
-
-        @free_ephemeral = []
-        cluster.ephemeral_datastores.each_value do |datastore|
-          @free_ephemeral << datastore.free_space
-        end
-
-        @free_persistent = []
-        cluster.persistent_datastores.each_value do |datastore|
-          @free_persistent << datastore.free_space
-        end
-
-        @free_shared = []
-        cluster.shared_datastores.each_value do |datastore|
-          @free_shared << datastore.free_space
-        end
-      end
-
       # Run the scoring function and return the placement score for the required
       # resources.
       #
+      # @param [Logging::Logger] logger logger to which to log.
+      # @param [Integer] requested_memory required memory.
+      # @param [Cluster] cluster requested cluster.
+      # @param [Integer] requested_ephemeral_size disk size in mb.
+      # @param [Array<Integer>] requested_persistent_sizes list of requested persistent sizes in mb.
       # @return [Integer] score.
-      def score
-        min_ephemeral = @ephemeral
-        min_persistent = @persistent.min
-        min_shared = min_ephemeral
-        if !min_persistent.nil? && min_persistent < min_shared
-          min_shared = min_persistent
-        end
-
-        # Filter out any datastores that are below the min threshold
-        filter(@free_ephemeral, min_ephemeral + DISK_THRESHOLD)
-        filter(@free_shared, min_shared + DISK_THRESHOLD)
-        unless @persistent.empty?
-          filter(@free_persistent, min_persistent + DISK_THRESHOLD)
-        end
+      def self.score(logger, cluster, requested_memory, requested_ephemeral_size, requested_persistent_sizes)
+        free_memory = cluster.free_memory
+        ephemeral_pool = DiskPool.new(cluster.ephemeral_datastores.values.map(&:free_space))
+        persistent_pool = DiskPool.new(cluster.persistent_datastores.values.map(&:free_space))
 
-        count = 0
+        successful_allocations = 0
         loop do
-          @free_memory -= @memory
-          if @free_memory < MEMORY_THRESHOLD
-            @logger.debug("#{@cluster.name} memory bound")
+          free_memory -= requested_memory
+          if free_memory < MEMORY_HEADROOM
+            logger.debug("#{cluster.name} memory bound")
             break
           end
 
-          consumed = consume_disk(@free_ephemeral, @ephemeral, min_ephemeral)
-          unless consumed
-            unless consume_disk(@free_shared, @ephemeral, min_shared)
-              @logger.debug("#{@cluster.name} ephemeral disk bound")
-              break
-            end
+          unless ephemeral_pool.consume_disk(requested_ephemeral_size)
+            logger.debug("#{cluster.name} ephemeral disk bound")
+            break
           end
 
-          unless @persistent.empty?
-            consumed_all = true
-            @persistent.each do |size|
-              consumed = consume_disk(@free_persistent, size, min_persistent)
-              unless consumed
-                unless consume_disk(@free_shared, size, min_shared)
-                  consumed_all = false
-                  @logger.debug("#{@cluster.name} persistent disk bound")
-                  break
-                end
-              end
+          unless requested_persistent_sizes.empty?
+            placed = requested_persistent_sizes.select { |size| persistent_pool.consume_disk(size) }
+            unless requested_persistent_sizes == placed
+              logger.debug("#{cluster.name} persistent disk bound")
+              break
             end
-            break unless consumed_all
           end
 
-          count += 1
+          successful_allocations += 1
         end
 
-        count
+        successful_allocations
       end
 
       private
 
-      # Filter out datastores from the pool that are below the free space
-      # threshold.
-      #
-      # @param [Array<Integer>] pool datastore pool.
-      # @param [Integer] threshold free space threshold
-      # @return [Array<Integer>] filtered pool.
-      def filter(pool, threshold)
-        pool.delete_if { |size| size < threshold }
-      end
+      class DiskPool
+        def initialize(sizes)
+          @sizes = sizes
+        end
 
-      # Consumes disk space from a datastore pool.
-      #
-      # @param [Array<Integer>] pool datastore pool.
-      # @param [Integer] size requested disk size.
-      # @param [Integer] min requested disk size, so the datastore can be
-      #   removed from the pool if it falls below this threshold.
-      # @return [true, false] boolean indicating that the disk space was
-      #   consumed.
-      def consume_disk(pool, size, min)
-        unless pool.empty?
-          pool.sort! { |a, b| b <=> a }
-          if pool[0] >= size + DISK_THRESHOLD
-            pool[0] -= size
-            pool.delete_at(0) if pool[0] < min + DISK_THRESHOLD
-            return true
+        # Consumes disk space from a datastore pool.
+        #
+        # @param [Integer] requested_size requested disk size.
+        # @return [true, false] boolean indicating that the disk space was consumed.
+        def consume_disk(requested_size)
+          unless @sizes.empty?
+            @sizes.sort! { |a, b| b <=> a }
+            if @sizes[0] >= requested_size + DISK_HEADROOM
+              @sizes[0] -= requested_size
+              return true
+            end
           end
+
+          false
         end
-        false
       end
     end
   end

data/lib/cloud/vsphere/resources/vm.rb

@@ -0,0 +1,185 @@
+module VSphereCloud
+  class Resources
+    class VM
+      include VimSdk
+      include RetryBlock
+
+      attr_reader :mob, :cid
+
+      def initialize(cid, mob, client, logger)
+        @client = client
+        @mob = mob
+        @cid = cid
+        @logger = logger
+      end
+
+      def inspect
+        "<VM: #{@mob} / #{@cid}>"
+      end
+
+      def cluster
+        cluster = cloud_searcher.get_properties(host_properties['parent'], Vim::ClusterComputeResource, 'name', ensure_all: true)
+        cluster['name']
+      end
+
+      def accessible_datastores
+        host_properties['datastore'].map do |store|
+          ds = cloud_searcher.get_properties(store, Vim::Datastore, 'info', ensure_all: true)
+          ds['info'].name
+        end
+      end
+
+      def datacenter
+        @client.find_parent(@mob, Vim::Datacenter)
+      end
+
+      def powered_on?
+        power_state == Vim::VirtualMachine::PowerState::POWERED_ON
+      end
+
+      def devices
+        properties['config.hardware.device']
+      end
+
+      def nics
+        devices.select { |device| device.kind_of?(Vim::Vm::Device::VirtualEthernetCard) }
+      end
+
+      def cdrom
+        devices.find { |device| device.kind_of?(Vim::Vm::Device::VirtualCdrom) }
+      end
+
+      def system_disk
+        devices.find { |device| device.kind_of?(Vim::Vm::Device::VirtualDisk) }
+      end
+
+      def persistent_disks
+       devices.select do |device|
+          device.kind_of?(Vim::Vm::Device::VirtualDisk) &&
+            device.backing.disk_mode == Vim::Vm::Device::VirtualDiskOption::DiskMode::INDEPENDENT_PERSISTENT
+        end
+      end
+
+      def pci_controller
+        devices.find { |device| device.kind_of?(Vim::Vm::Device::VirtualPCIController) }
+      end
+
+      def fix_device_unit_numbers(device_changes)
+        controllers_available_unit_numbers = Hash.new { |h,k| h[k] = (0..15).to_a }
+        devices.each do |device|
+          if device.controller_key
+            available_unit_numbers = controllers_available_unit_numbers[device.controller_key]
+            available_unit_numbers.delete(device.unit_number)
+          end
+        end
+
+        device_changes.each do |device_change|
+          device = device_change.device
+          if device.controller_key && device.unit_number.nil?
+            available_unit_numbers = controllers_available_unit_numbers[device.controller_key]
+            raise "No available unit numbers for device: #{device.inspect}" if available_unit_numbers.empty?
+            device.unit_number = available_unit_numbers.shift
+          end
+        end
+      end
+
+      def shutdown
+        @logger.debug('Waiting for the VM to shutdown')
+        begin
+          begin
+            @mob.shutdown_guest
+          rescue => e
+            @logger.debug("Ignoring possible race condition when a VM has powered off by the time we ask it to shutdown: #{e.inspect}")
+          end
+
+          wait_until_off(60)
+        rescue VSphereCloud::Cloud::TimeoutException
+          @logger.debug('The guest did not shutdown in time, requesting it to power off')
+          @client.power_off_vm(@mob)
+        end
+      end
+
+      def power_off
+        retry_block do
+          question = properties['runtime.question']
+          if question
+            choices = question.choice
+            @logger.info("VM is blocked on a question: #{question.text}, " +
+              "providing default answer: #{choices.choice_info[choices.default_index].label}")
+            @client.answer_vm(@mob, question.id, choices.choice_info[choices.default_index].key)
+            power_state = cloud_searcher.get_property(@mob, Vim::VirtualMachine, 'runtime.powerState')
+          else
+            power_state = properties['runtime.powerState']
+          end
+
+          if power_state != Vim::VirtualMachine::PowerState::POWERED_OFF
+            @logger.info("Powering off vm: #{@cid}")
+            @client.power_off_vm(@mob)
+          end
+        end
+      end
+
+      def disk_by_cid(disk_cid)
+        devices.find do |d|
+          d.kind_of?(Vim::Vm::Device::VirtualDisk) &&
+            d.backing.file_name.end_with?("/#{disk_cid}.vmdk")
+        end
+      end
+
+      def reboot
+        @mob.reboot_guest
+      end
+
+      def power_on
+        @client.power_on_vm(datacenter, @mob)
+      end
+
+      def delete
+        retry_block { @client.delete_vm(@mob) }
+      end
+
+      def reload
+        @properties = nil
+        @host_properties = nil
+      end
+
+      def wait_until_off(timeout)
+        started = Time.now
+        loop do
+          power_state = cloud_searcher.get_property(@mob, Vim::VirtualMachine, 'runtime.powerState')
+          break if power_state == Vim::VirtualMachine::PowerState::POWERED_OFF
+          raise VSphereCloud::Cloud::TimeoutException if Time.now - started > timeout
+          sleep(1.0)
+        end
+      end
+
+      private
+
+      def power_state
+        properties['runtime.powerState']
+      end
+
+      def properties
+        @properties ||= cloud_searcher.get_properties(
+          @mob,
+          Vim::VirtualMachine,
+          ['runtime.powerState', 'runtime.question', 'config.hardware.device', 'name', 'runtime'],
+          ensure: ['config.hardware.device', 'runtime']
+        )
+      end
+
+      def host_properties
+        @host_properties ||= cloud_searcher.get_properties(
+          properties['runtime'].host,
+          Vim::HostSystem,
+          ['datastore', 'parent'],
+          ensure_all: true
+        )
+      end
+
+      def cloud_searcher
+        @client.cloud_searcher
+      end
+    end
+  end
+end

data/lib/cloud/vsphere/resources.rb

@@ -2,14 +2,15 @@ require 'cloud/vsphere/resources/datacenter'
 
 module VSphereCloud
   class Resources
-    MEMORY_THRESHOLD = 128
-    DISK_THRESHOLD = 1024
+    MEMORY_HEADROOM = 128
+    DISK_HEADROOM = 1024
     STALE_TIMEOUT = 60
     BYTES_IN_MB = 1024 * 1024
 
     attr_reader :drs_rules
 
-    def initialize(config)
+    def initialize(datacenter, config)
+      @datacenter = datacenter
       @config = config
       @logger = config.logger
       @last_update = 0
@@ -17,128 +18,111 @@ module VSphereCloud
       @drs_rules = []
     end
 
-    # Returns the list of datacenters available for placement.
-    #
-    # Will lazily load them and reload the data when it's stale.
-    #
-    # @return [List<Resources::Datacenter>] datacenters.
-    def datacenters
-      @lock.synchronize do
-        update if Time.now.to_i - @last_update > STALE_TIMEOUT
-      end
-      @datacenters
-    end
-
-    # Returns the persistent datastore for the requested context.
-    #
-    # @param [String] dc_name datacenter name.
-    # @param [String] cluster_name cluster name.
-    # @param [String] datastore_name datastore name.
-    # @return [Resources::Datastore] persistent datastore.
-    def persistent_datastore(dc_name, cluster_name, datastore_name)
-      datacenter = datacenters[dc_name]
-      return nil if datacenter.nil?
-      cluster = datacenter.clusters[cluster_name]
-      return nil if cluster.nil?
-      cluster.persistent(datastore_name)
-    end
-
-    # Validate that the persistent datastore is still valid so we don't have to
-    # move the disk.
-    #
-    # @param [String] dc_name datacenter name.
-    # @param [String] datastore_name datastore name.
-    # @return [true, false] true iff the datastore still exists and is in the
-    #   persistent pool.
-    def validate_persistent_datastore(dc_name, datastore_name)
-      datacenter = datacenters[dc_name]
-      if datacenter.nil?
-        raise "Invalid datacenter #{dc_name} #{datacenters.inspect}"
-      end
-      datacenter.clusters.each_value do |cluster|
-        return true unless cluster.persistent(datastore_name).nil?
-      end
-      false
-    end
-
     # Place the persistent datastore in the given datacenter and cluster with
     # the requested disk space.
     #
-    # @param [String] dc_name datacenter name.
     # @param [String] cluster_name cluster name.
-    # @param [Integer] disk_space disk space.
+    # @param [Integer] disk_size_in_mb disk size in mb.
     # @return [Datastore?] datastore if it was placed succesfuly.
-    def place_persistent_datastore(dc_name, cluster_name, disk_space)
+    def pick_persistent_datastore_in_cluster(cluster_name, disk_size_in_mb)
       @lock.synchronize do
-        datacenter = datacenters[dc_name]
-        return nil if datacenter.nil?
-        cluster = datacenter.clusters[cluster_name]
+        cluster = @datacenter.clusters[cluster_name]
         return nil if cluster.nil?
-        datastore = cluster.pick_persistent(disk_space)
-        return nil if datastore.nil?
-        datastore.allocate(disk_space)
-        return datastore
+
+        pick_datastore(cluster, disk_size_in_mb)
       end
     end
 
-    # Find a place for the requested resources.
+    # Find a cluster for a vm with the requested memory and ephemeral storage, attempting
+    # to allocate it near existing persistent disks.
     #
-    # @param [Integer] memory requested memory.
-    # @param [Integer] ephemeral requested ephemeral storage.
-    # @param [Array<Hash>] persistent requested persistent storage.
-    # @return [Array] an array/tuple of Cluster and Datastore if the resources
-    #   were placed successfully, otherwise exception.
-    def place(memory, ephemeral, persistent)
-      populate_resources(persistent)
-
-      # calculate locality to prioritizing clusters that contain the most
-      # persistent data.
-      locality = cluster_locality(persistent)
-      locality.sort! { |a, b| b[1] <=> a[1] }
-
+    # @param [Integer] requested_memory_in_mb requested memory.
+    # @param [Integer] requested_ephemeral_disk_size_in_mb requested ephemeral storage.
+    # @param [Array<Resources::Disk>] existing_persistent_disks existing persistent disks, if any.
+    # @return [Cluster] selected cluster if the resources were placed successfully, otherwise raises.
+    def pick_cluster_for_vm(requested_memory_in_mb, requested_ephemeral_disk_size_in_mb, existing_persistent_disks)
       @lock.synchronize do
-        locality.each do |cluster, _|
-          persistent_sizes = persistent_sizes_for_cluster(cluster, persistent)
-
-          scorer = Scorer.new(@config, cluster, memory, ephemeral, persistent_sizes)
-          if scorer.score > 0
-            datastore = cluster.pick_ephemeral(ephemeral)
-            if datastore
-              cluster.allocate(memory)
-              datastore.allocate(ephemeral)
-              return [cluster, datastore]
-            end
+        # calculate locality to prioritizing clusters that contain the most persistent data.
+        clusters = @datacenter.clusters.values
+        persistent_disk_index = PersistentDiskIndex.new(clusters, existing_persistent_disks)
+
+        scored_clusters = clusters.map do |cluster|
+          persistent_disk_not_in_this_cluster = existing_persistent_disks.reject do |disk|
+            persistent_disk_index.clusters_connected_to_disk(disk).include?(cluster)
           end
+
+          score = Scorer.score(
+            @config.logger,
+            cluster,
+            requested_memory_in_mb,
+            requested_ephemeral_disk_size_in_mb,
+            persistent_disk_not_in_this_cluster.map(&:size_in_mb)
+          )
+
+          [cluster, score]
         end
 
-        unless locality.empty?
-          @logger.debug("Ignoring datastore locality as we could not find " +
-                          "any resources near disks: #{persistent.inspect}")
+        acceptable_clusters = scored_clusters.select { |_, score| score > 0 }
+
+        @logger.debug("Acceptable clusters: #{acceptable_clusters.inspect}")
+
+        if acceptable_clusters.empty?
+          total_persistent_size = existing_persistent_disks.map(&:size_in_mb).inject(0, :+)
+          cluster_infos = clusters.map { |cluster| describe_cluster(cluster) }
+
+          raise "Unable to allocate vm with #{requested_memory_in_mb}mb RAM, " +
+              "#{requested_ephemeral_disk_size_in_mb / 1024}gb ephemeral disk, " +
+              "and #{total_persistent_size / 1024}gb persistent disk from any cluster.\n#{cluster_infos.join(", ")}."
         end
 
-        weighted_clusters = []
-        datacenter = datacenters.first.last
-        datacenter.clusters.each_value do |cluster|
-          persistent_sizes = persistent_sizes_for_cluster(cluster, persistent)
-          scorer = Scorer.new(@config, cluster, memory, ephemeral, persistent_sizes)
-          score = scorer.score
-          @logger.debug("Score: #{cluster.name}: #{score}")
-          weighted_clusters << [cluster, score] if score > 0
+        acceptable_clusters = acceptable_clusters.sort_by do |cluster, _score|
+          persistent_disk_index.disks_connected_to_cluster(cluster).map(&:size_in_mb).inject(0, :+)
+        end.reverse
+
+        if acceptable_clusters.any? { |cluster, _| persistent_disk_index.disks_connected_to_cluster(cluster).any? }
+          @logger.debug('Choosing cluster with the greatest available disk')
+          selected_cluster, _ = acceptable_clusters.first
+        else
+          @logger.debug('Choosing cluster by weighted random')
+          selected_cluster = Util.weighted_random(acceptable_clusters)
         end
 
-        raise "No available resources" if weighted_clusters.empty?
+        @logger.debug("Selected cluster '#{selected_cluster.name}'")
+
+        selected_cluster.allocate(requested_memory_in_mb)
+        selected_cluster
+      end
+    end
+
+    def describe_cluster(cluster)
+      "#{cluster.name} has #{cluster.free_memory}mb/" +
+        "#{cluster.total_free_ephemeral_disk_in_mb / 1024}gb/" +
+        "#{cluster.total_free_persistent_disk_in_mb / 1024}gb"
+    end
 
-        cluster = Util.weighted_random(weighted_clusters)
+    def pick_ephemeral_datastore(cluster, disk_size_in_mb)
+      @lock.synchronize do
+        datastore = cluster.pick_ephemeral(disk_size_in_mb)
+        if datastore.nil?
+          raise Bosh::Clouds::NoDiskSpace.new(
+              "Not enough ephemeral disk space (#{disk_size_in_mb}MB) in cluster #{cluster.name}")
+        end
 
-        datastore = cluster.pick_ephemeral(ephemeral)
+        datastore.allocate(disk_size_in_mb)
+        datastore
+      end
+    end
 
-        if datastore
-          cluster.allocate(memory)
-          datastore.allocate(ephemeral)
-          return [cluster, datastore]
+    def pick_persistent_datastore(cluster, disk_size_in_mb)
+      @lock.synchronize do
+        datastore = cluster.pick_persistent(disk_size_in_mb)
+        if datastore.nil?
+          raise Bosh::Clouds::NoDiskSpace.new(
+              "Not enough persistent disk space (#{disk_size_in_mb}MB) in cluster #{cluster.name}")
         end
 
-        raise "No available resources"
+        datastore.allocate(disk_size_in_mb)
+        datastore
       end
     end
 
@@ -146,74 +130,35 @@ module VSphereCloud
 
     attr_reader :config
 
-    # Updates the resource models from vSphere.
-    # @return [void]
-    def update
-      #datacenter_config = config.vcenter_datacenter
-      datacenter = Datacenter.new(config)
-      @datacenters = { datacenter.name => datacenter }
-      @last_update = Time.now.to_i
+    def pick_datastore(cluster, disk_space)
+      datastore = cluster.pick_persistent(disk_space)
+      return nil if datastore.nil?
+      datastore.allocate(disk_space)
+      datastore
     end
 
-    # Calculates the cluster locality for the provided persistent disks.
-    #
-    # @param [Array<Hash>] disks persistent disk specs.
-    # @return [Hash<String, Integer>] hash of cluster names to amount of
-    #   persistent disk space is currently allocated on them.
-    def cluster_locality(disks)
-      locality = {}
-      disks.each do |disk|
-        cluster = disk[:cluster]
-        unless cluster.nil?
-          locality[cluster] ||= 0
-          locality[cluster] += disk[:size]
-        end
+    class PersistentDiskIndex
+      def initialize(clusters, existing_persistent_disks)
+        @clusters_to_disks = Hash[*clusters.map do |cluster|
+            [cluster, existing_persistent_disks.select { |disk| cluster_includes_datastore?(cluster, disk.datastore) }]
+          end.flatten(1)]
+
+        @disks_to_clusters = Hash[*existing_persistent_disks.map do |disk|
+            [disk, clusters.select { |cluster| cluster_includes_datastore?(cluster, disk.datastore) }]
+          end.flatten(1)]
       end
-      locality.to_a
-    end
 
-    # Fill in the resource models on the provided persistent disk specs.
-    # @param [Array<Hash>] disks persistent disk specs.
-    # @return [void]
-    def populate_resources(disks)
-      disks.each do |disk|
-        unless disk[:ds_name].nil?
-          resources = persistent_datastore_resources(disk[:dc_name],
-                                                     disk[:ds_name])
-          if resources
-            disk[:datacenter], disk[:cluster], disk[:datastore] = resources
-          end
-        end
+      def cluster_includes_datastore?(cluster, datastore)
+        cluster.persistent(datastore.name) != nil
       end
-    end
 
-    # Find the resource models for a given datacenter and datastore name.
-    #
-    # Has to traverse the resource hierarchy to find the cluster, then returns
-    # all of the resources.
-    #
-    # @param [String] dc_name datacenter name.
-    # @param [String] ds_name datastore name.
-    # @return [Array] array/tuple of Datacenter, Cluster, and Datastore.
-    def persistent_datastore_resources(dc_name, ds_name)
-      datacenter = datacenters[dc_name]
-      return nil if datacenter.nil?
-      datacenter.clusters.each_value do |cluster|
-        datastore = cluster.persistent(ds_name)
-        return [datacenter, cluster, datastore] unless datastore.nil?
+      def disks_connected_to_cluster(cluster)
+        @clusters_to_disks[cluster]
       end
-      nil
-    end
 
-    # Filters out all of the persistent disk specs that were already allocated
-    # in the cluster.
-    #
-    # @param [Resources::Cluster] cluster specified cluster.
-    # @param [Array<Hash>] disks disk specs.
-    # @return [Array<Hash>] filtered out disk specs.
-    def persistent_sizes_for_cluster(cluster, disks)
-      disks.select { |disk| disk[:cluster] != cluster }.
-        collect { |disk| disk[:size] }
+      def clusters_connected_to_disk(disk)
+        @disks_to_clusters[disk]
+      end
     end
   end
 end