bosh_vsphere_cpi 2.1.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73c2f8de1b6a30b2e09442508f67a4429defaad3
-  data.tar.gz: 8b6973f59b1e9df8353d4a530c236fe5b9ff0cfb
+  metadata.gz: 20484247b0bb76cefc40f2f28054428a6680991b
+  data.tar.gz: 1d48fb046acb02cf0417b6b803c28e6883291b0a
 SHA512:
-  metadata.gz: ea07e43ec4a22772feab1ebd4bab1e30b7ffb66c42b84005ada82d12e36fd7fc521418685f539f48d183752387453ab8e4d9dc9072c79f4bb5a6c6fdeca22b8b
-  data.tar.gz: ebe07d93fcb600539b6aeb6a778441159897c1b6661f564ba484726836d7eb5458c3219e25ed3d2f0ec20248d65a2aaf66c15781132cdf8ec74fba11a02e26f1
+  metadata.gz: 0ca9d5f7600ad20b84ce67c8ddeab6381c7d77ffa2baaff0323f492bfb267e57570385249b91f0c49970ddbb75b98c453b6422e02e5abd5c443f0e7f42d91627
+  data.tar.gz: 17673951c1c87149dc4e32ba21627f1d139a96af603d81cdd4deace7791dbcdf7a4cf341f9ec7ba0f5b825bb300e33ce3bd147beb60b2bc2cd387da756938bb1

data/lib/cloud/vsphere/client.rb

@@ -13,7 +13,7 @@ module VSphereCloud
     def initialize(host, options={})
       @soap_stub = SoapStub.new(host, options[:soap_log]).create
 
-      @service_instance =Vim::ServiceInstance.new('ServiceInstance', @soap_stub)
+      @service_instance = Vim::ServiceInstance.new('ServiceInstance', @soap_stub)
       @service_content = @service_instance.content
 
       @metrics_cache  = {}
@@ -130,6 +130,31 @@ module VSphereCloud
       @service_content.search_index.find_by_inventory_path(full_path)
     end
 
+    def find_vm_by_ip(ip)
+      @service_content.search_index.find_by_ip(nil, ip, true)
+    end
+
+    def find_vm_by_name(datacenter, vm_name)
+      yield_all_resources_by_name(datacenter, 'VirtualMachine') do |vm_mob, name|
+        if name == vm_name
+          return vm_mob
+        end
+      end
+
+      nil
+    end
+
+    def find_all_stemcell_replicas(datacenter, stemcell_id)
+      matches = []
+      yield_all_resources_by_name(datacenter, 'VirtualMachine') do |vm_mob, name|
+        if name =~ Regexp.new(stemcell_id)
+          matches << vm_mob
+        end
+      end
+
+      matches
+    end
+
     def wait_for_task(task)
       interval = 1.0
       started = Time.now
@@ -208,13 +233,17 @@ module VSphereCloud
       disk_size_in_mb.nil? ? nil : Resources::Disk.new(disk_cid, disk_size_in_mb, datastore, disk_path)
     end
 
-    def create_disk(datacenter, datastore, disk_cid, disk_folder, disk_size_in_mb)
+    def create_disk(datacenter, datastore, disk_cid, disk_folder, disk_size_in_mb, disk_type)
+      if disk_type.nil?
+        raise 'no disk type specified'
+      end
+
       disk_path = "[#{datastore.name}] #{disk_folder}/#{disk_cid}.vmdk"
 
       create_parent_folder(datacenter, disk_path)
 
       disk_spec = VimSdk::Vim::VirtualDiskManager::FileBackedVirtualDiskSpec.new
-      disk_spec.disk_type = 'preallocated'
+      disk_spec.disk_type = disk_type
       disk_spec.capacity_kb = disk_size_in_mb * 1024
       disk_spec.adapter_type = 'lsiLogic'
 
@@ -296,5 +325,55 @@ module VSphereCloud
       metrics.each { |metric| result[metric_names[metric.counter_id]] = metric }
       result
     end
+
+    def create_filter_spec(datacenter, resource_type, property_names)
+      # Create a view to list all the resources in the root folder
+      container_view = @service_content.view_manager.create_container_view(datacenter.mob, [resource_type], true)
+
+      # Create an object spec to define the beginning of the traversal
+      object_spec = VimSdk::Vmodl::Query::PropertyCollector::ObjectSpec.new
+      object_spec.obj = container_view
+      object_spec.skip = false
+
+      # Create a traversal spec to select all objects in the 'view' of the ContainerView
+      vm_traversal_spec = VimSdk::Vmodl::Query::PropertyCollector::TraversalSpec.new
+      vm_traversal_spec.name = 'searchTraversalSpec'
+      vm_traversal_spec.path = 'view'
+      vm_traversal_spec.type = 'ContainerView'
+      object_spec.select_set << vm_traversal_spec
+
+      # Specify the properties for retrieval
+      property_spec = VimSdk::Vmodl::Query::PropertyCollector::PropertySpec.new
+      property_spec.type = resource_type
+      property_spec.path_set.concat(property_names)
+      property_spec.all = false
+
+      filter_spec = VimSdk::Vmodl::Query::PropertyCollector::FilterSpec.new
+      filter_spec.object_set << object_spec
+      filter_spec.prop_set << property_spec
+      filter_spec
+    end
+
+    def yield_all_resources_by_name(datacenter, type)
+      raise 'Requires a vSphere object type' if type.nil?
+      # Collect all the 'name' attributes of all VMs in the datacenter
+      vm_filter_spec = create_filter_spec(datacenter, type, ['name'])
+      ro = VimSdk::Vmodl::Query::PropertyCollector::RetrieveOptions.new
+      result = @service_content.property_collector.retrieve_properties_ex([vm_filter_spec], ro)
+
+      # Retrieve Properties Ex returns paginated results, so we continue retrieving
+      # more pages if the token indicates there are more pages.
+      token = "not nil"
+      until token.nil?
+        result.objects.each do |object_content|
+          # Yield the object MOB and its name
+          yield object_content.obj, object_content.prop_set.first.val
+        end
+        token = result.token
+        unless token.nil?
+          result = @service_content.property_collector.continue_retrieve_properties_ex(token)
+        end
+      end
+    end
   end
 end

data/lib/cloud/vsphere/cloud.rb

@@ -133,25 +133,24 @@ module VSphereCloud
       with_thread_name("delete_stemcell(#{stemcell})") do
         Bosh::ThreadPool.new(max_threads: 32, logger: @logger).wrap do |pool|
           @logger.info("Looking for stemcell replicas in: #{@datacenter.name}")
-          templates = @cloud_searcher.get_property(@datacenter.template_folder.mob, Vim::Folder, 'childEntity', ensure_all: true)
-          template_properties = @cloud_searcher.get_properties(templates, Vim::VirtualMachine, ['name'])
-          template_properties.each_value do |properties|
-            template_name = properties['name'].gsub('%2f', '/')
-            if template_name.split('/').first.strip == stemcell
-              @logger.info("Found: #{template_name}")
-              pool.process do
-                @logger.info("Deleting: #{template_name}")
-                client.delete_vm(properties[:obj])
-                @logger.info("Deleted: #{template_name}")
-              end
+          matches = client.find_all_stemcell_replicas(@datacenter, stemcell)
+
+          matches.each do |sc|
+            sc_name = sc.name
+            @logger.info("Found: #{sc_name}")
+            pool.process do
+              @logger.info("Deleting: #{sc_name}")
+              client.delete_vm(sc)
+              @logger.info("Deleted: #{sc_name}")
             end
           end
+
         end
       end
     end
 
     def stemcell_vm(name)
-      client.find_by_inventory_path([@datacenter.name, 'vm', @datacenter.template_folder.path_components, name])
+      client.find_vm_by_name(@datacenter, name)
     end
 
     def create_vm(agent_id, stemcell, cloud_properties, networks, disk_locality = nil, environment = nil)
@@ -381,7 +380,8 @@ module VSphereCloud
           cluster = @datacenter.clusters[vm.cluster]
         end
 
-        disk = disk_provider.create(size_in_mb, cluster)
+        disk_type = cloud_properties['type']
+        disk = disk_provider.create(size_in_mb, disk_type, cluster)
         @logger.info("Created disk: #{disk.inspect}")
         disk.cid
       end
@@ -398,21 +398,18 @@ module VSphereCloud
     end
 
     def replicate_stemcell(cluster, datastore, stemcell)
-      stemcell_vm_path_components = [cluster.datacenter.name, 'vm', cluster.datacenter.template_folder.path_components, stemcell]
-      stemcell_vm = client.find_by_inventory_path(stemcell_vm_path_components)
-      raise "Could not find VM for stemcell '#{stemcell}' at path '#{stemcell_vm_path_components.join("/")}'" if stemcell_vm.nil?
+      stemcell_vm = client.find_vm_by_name(@datacenter, stemcell)
+      raise "Could not find VM for stemcell '#{stemcell}'" if stemcell_vm.nil?
       stemcell_datastore = @cloud_searcher.get_property(stemcell_vm, Vim::VirtualMachine, 'datastore', ensure_all: true)
 
       if stemcell_datastore != datastore.mob
         @logger.info("Stemcell lives on a different datastore, looking for a local copy of: #{stemcell}.")
         local_stemcell_name = "#{stemcell} %2f #{datastore.mob.__mo_id__}"
-        local_stemcell_path =
-          [cluster.datacenter.name, 'vm', cluster.datacenter.template_folder.path_components, local_stemcell_name]
-        replicated_stemcell_vm = client.find_by_inventory_path(local_stemcell_path)
 
+        replicated_stemcell_vm = client.find_vm_by_name(@datacenter, local_stemcell_name)
         if replicated_stemcell_vm.nil?
           @logger.info("Cluster doesn't have stemcell #{stemcell}, replicating")
-            replicated_stemcell_vm = replicate_stemcell_helper(stemcell, stemcell_vm, local_stemcell_name, cluster, datastore, replicated_stemcell_vm, local_stemcell_path)
+          replicated_stemcell_vm = replicate_stemcell_helper(stemcell, stemcell_vm, local_stemcell_name, cluster, datastore, replicated_stemcell_vm)
         else
           @logger.info("Found local stemcell replica: #{replicated_stemcell_vm}")
         end
@@ -704,7 +701,7 @@ module VSphereCloud
       placer.nil? ? @resources : placer
     end
 
-  def replicate_stemcell_helper(stemcell, stemcell_vm, local_stemcell_name, cluster, datastore, replicated_stemcell_vm, local_stemcell_path)
+  def replicate_stemcell_helper(stemcell, stemcell_vm, local_stemcell_name, cluster, datastore, replicated_stemcell_vm)
     @logger.info("Replicating #{stemcell} (#{stemcell_vm}) to #{local_stemcell_name}")
     task = clone_vm(stemcell_vm,
                     local_stemcell_name,
@@ -716,6 +713,8 @@ module VSphereCloud
       @logger.info("Replicated #{stemcell} (#{stemcell_vm}) to #{local_stemcell_name} (#{replicated_stemcell_vm})")
     rescue VSphereCloud::Client::DuplicateName => ex
       @logger.info("Stemcell is being replicated by another thread, waiting for #{local_stemcell_name} to be ready")
+      local_stemcell_path =
+        [cluster.datacenter.name, 'vm', cluster.datacenter.template_folder.path_components, local_stemcell_name]
       replicated_stemcell_vm = client.find_by_inventory_path(local_stemcell_path)
       # get_properties will ensure the existence of the snapshot by retrying.
       # This forces us to wait for a valid snapshot before returning with the

data/lib/cloud/vsphere/disk_provider.rb

@@ -3,6 +3,16 @@ require 'cloud/vsphere/resources/disk'
 
 module VSphereCloud
   class DiskProvider
+  # https://pubs.vmware.com/vsphere-55/index.jsp?topic=%2Fcom.vmware.wssdk.apiref.doc%2Fvim.VirtualDiskManager.VirtualDiskType.html
+  SUPPORTED_DISK_TYPES = %w{
+      eagerZeroedThick
+      preallocated
+      thick
+      thin
+    }
+
+    DEFAULT_DISK_TYPE = 'preallocated'
+
     def initialize(virtual_disk_manager, datacenter, resources, disk_path, client, logger)
       @virtual_disk_manager = virtual_disk_manager
       @datacenter = datacenter
@@ -12,7 +22,16 @@ module VSphereCloud
       @logger = logger
     end
 
-    def create(disk_size_in_mb, cluster)
+    def create(disk_size_in_mb, disk_type, cluster)
+      type = disk_type
+      if type.nil?
+        type = DEFAULT_DISK_TYPE
+      end
+
+      unless SUPPORTED_DISK_TYPES.include?(type)
+        raise "Disk type: '#{disk_type}' is not supported"
+      end
+
       if cluster
         datastore = @resources.pick_persistent_datastore_in_cluster(cluster.name, disk_size_in_mb)
       else
@@ -21,14 +40,17 @@ module VSphereCloud
       disk_cid = "disk-#{SecureRandom.uuid}"
       @logger.debug("Creating disk '#{disk_cid}' in datastore '#{datastore.name}'")
 
-      @client.create_disk(@datacenter, datastore, disk_cid, @disk_path, disk_size_in_mb)
+      @client.create_disk(@datacenter, datastore, disk_cid, @disk_path, disk_size_in_mb, type)
     end
 
     def find_and_move(disk_cid, cluster, datacenter, accessible_datastores)
       disk = find(disk_cid)
       disk_in_persistent_datastore = @datacenter.persistent_datastores.include?(disk.datastore.name)
       disk_in_accessible_datastore = accessible_datastores.include?(disk.datastore.name)
-      return disk if disk_in_persistent_datastore && disk_in_accessible_datastore
+      if disk_in_persistent_datastore && disk_in_accessible_datastore
+        @logger.info("Disk #{disk_cid} found in an accessible, persistent datastore '#{disk.datastore.name}'")
+        return disk
+      end
 
       destination_datastore = @resources.pick_persistent_datastore_in_cluster(cluster.name, disk.size_in_mb)
 

data/lib/cloud/vsphere/ip_conflict_detector.rb

@@ -0,0 +1,41 @@
+module VSphereCloud
+  class IPConflictDetector
+
+    def initialize(logger, client, networks)
+      @logger = logger
+      @client = client
+      @networks = networks
+      @desired_ip_mapping = []
+
+      @networks.map do |_, network_spec|
+        ip = network_spec['ip']
+        network_name = network_spec.fetch('cloud_properties', [])['name']
+
+        if ip && network_name
+          @desired_ip_mapping << {ip: ip, name: network_name}
+        end
+      end
+    end
+
+    def conflicts
+      conflicts = []
+      @desired_ip_mapping.each do |mapping|
+        @logger.info("Checking if ip '#{mapping[:ip]}' is in use")
+        vm = @client.find_vm_by_ip(mapping[:ip])
+        if vm.nil?
+          next
+        end
+
+        vm.guest.net.each do |nic|
+          if nic.ip_address.include?(mapping[:ip]) && nic.network == mapping[:name]
+            @logger.info("found conflicting vm: #{vm.name}, on network: #{mapping[:name]} with ip: #{mapping[:ip]}")
+            conflicts << {vm_name: vm.name, network_name: mapping[:name], ip: mapping[:ip]}
+          end
+        end
+      end
+
+      conflicts
+    end
+
+  end
+end

data/lib/cloud/vsphere/resources/cluster.rb

@@ -139,7 +139,7 @@ module VSphereCloud
         available_datastores = datastores.reject { |datastore| datastore.free_space - size < DISK_HEADROOM }
 
         @logger.debug("Looking for a #{type} datastore in #{self.name} with #{size}MB free space.")
-        @logger.debug("All datastores: #{datastores.map(&:debug_info)}")
+        @logger.debug("All datastores within cluster #{self.name}: #{datastores.map(&:debug_info)}")
         @logger.debug("Datastores with enough space: #{available_datastores.map(&:debug_info)}")
 
         selected_datastore = Util.weighted_random(available_datastores.map { |datastore| [datastore, datastore.free_space] })

data/lib/cloud/vsphere/resources/datacenter.rb

@@ -68,12 +68,16 @@ module VSphereCloud
       end
 
       def clusters
-        clusters = {}
-        @clusters.each do |cluster_name, cluster_config|
-          clusters[cluster_name] = @cluster_provider.find(cluster_name, cluster_config)
-        end
         @logger.debug("All clusters provided: #{@clusters}")
-        clusters
+        @clusters.keys.inject({}) do |acc, cluster_name|
+          acc[cluster_name] = find_cluster(cluster_name)
+          acc
+        end
+      end
+
+      def find_cluster(cluster_name)
+        cluster_config = @clusters[cluster_name]
+        @cluster_provider.find(cluster_name, cluster_config)
       end
 
       def persistent_datastores
@@ -103,7 +107,7 @@ module VSphereCloud
         available_datastores = datastores.reject { |datastore| datastore.free_space - size < DISK_HEADROOM }
 
         @logger.debug("Looking for a '#{type}' datastore with #{size}MB free space.")
-        @logger.debug("All datastores: #{datastores.map(&:debug_info)}")
+        @logger.debug("All datastores within datacenter #{self.name}: #{datastores.map(&:debug_info)}")
         @logger.debug("Datastores with enough space: #{available_datastores.map(&:debug_info)}")
 
         selected_datastore = Util.weighted_random(available_datastores.map { |datastore| [datastore, datastore.free_space] })

data/lib/cloud/vsphere/vm_creator.rb

@@ -18,6 +18,11 @@ module VSphereCloud
     end
 
     def create(agent_id, stemcell_cid, networks, persistent_disk_cids, environment)
+      ip_conflicts = IPConflictDetector.new(@logger, @client, networks).conflicts
+      unless ip_conflicts.empty?
+        raise "Cannot create new VM because of IP conflicts with other VMs on the same networks: #{ip_conflicts}"
+      end
+
       stemcell_vm = @cpi.stemcell_vm(stemcell_cid)
       raise "Could not find VM for stemcell '#{stemcell_cid}'" if stemcell_vm.nil?
 

data/lib/cloud/vsphere/vm_provider.rb

@@ -7,8 +7,8 @@ module VSphereCloud
     end
 
     def find(vm_cid)
-      vm_mob = @client.find_by_inventory_path(@datacenter.vm_path(vm_cid))
-      raise Bosh::Clouds::VMNotFound, "VM '#{vm_cid}' not found at path '#{@datacenter.vm_path(vm_cid)}'" if vm_mob.nil?
+      vm_mob = @client.find_vm_by_name(@datacenter, vm_cid)
+      raise Bosh::Clouds::VMNotFound, "VM '#{vm_cid}' not found in datacenter '#{@datacenter.name}'" if vm_mob.nil?
 
       Resources::VM.new(vm_cid, vm_mob, @client, @logger)
     end

data/lib/cloud/vsphere.rb

@@ -31,6 +31,7 @@ require 'cloud/vsphere/resources/util'
 require 'cloud/vsphere/soap_stub'
 require 'cloud/vsphere/vm_creator_builder'
 require 'cloud/vsphere/vm_provider'
+require 'cloud/vsphere/ip_conflict_detector'
 
 module Bosh
   module Clouds

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bosh_vsphere_cpi
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - VMware
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-02 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bosh_common
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.7.1
+        version: 2.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.7.1
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   name: mono_logger
   requirement: !ruby/object:Gem::Requirement
@@ -131,6 +131,7 @@ files:
 - lib/cloud/vsphere/drs_rules/vm_attribute_manager.rb
 - lib/cloud/vsphere/file_provider.rb
 - lib/cloud/vsphere/fixed_cluster_placer.rb
+- lib/cloud/vsphere/ip_conflict_detector.rb
 - lib/cloud/vsphere/lease_obtainer.rb
 - lib/cloud/vsphere/lease_updater.rb
 - lib/cloud/vsphere/object_stringifier.rb
@@ -198,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: BOSH VSphere CPI