fog-vsphere 2.3.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2802cab39cda0d34fbf317a0ef39e79b53a7ce20fd736e2f41ac86e0d95f494b
-  data.tar.gz: d173548894fdfed4a4c517e1e6b30b513420b2bf59be77ce1147350ba695a15f
+  metadata.gz: d5281df844dc5370c7eeb4aa1b6d062edc780650cfa26143ef122a9f719b1305
+  data.tar.gz: ddd9c2dfe8393e74aa74267406f23638047f52ae426dcb98b30d0db64666e5d2
 SHA512:
-  metadata.gz: 62ec056eb0f333a9a80eb1d4ec523cb2e5f8893d8b0cdd9f5243210a920b2acfeeaa74dd5fe00770b5eb77bd5a255167dde730b8f89b257e1d8891133803e1e6
-  data.tar.gz: 4ac64f5bca359eb6e6290d137e1975644929f4eee39e0ad5f48d6c7fb2f58d8f6a235d557efe5bb26f1e0bfc98f88321a199b78bbf6b195aa92c3459e7b9bfd9
+  metadata.gz: cf18dc5cee08a4be3919c5473237ad83ba609d95868891dd7afb43d8d0aacd4d89ff05f211e3a89e25610aa3b5ee0da75f846fbbfdd7a3d1e3b9301d860f7586
+  data.tar.gz: 597988c88827a950b9ff4ab4363c344f72ebca5833cae3397358ddab8fae9f4f63813c8fa54826c3c2956e0469898165e582f4ef17406c22bd95d1b9c1fd34fd

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## v2.4.0
+* improve folder loading performance (#158)
+* improve storage pod loading performance (#160)
+* improve datastore loading performance (#159)
+* prevent hard fail on undefined legacy networks (#157)
+* add support to create vms on NSX-T networks (#153)
+* set the server for fog volume objects (#152)
+* add server guard to unit_number calculation (#152)
+
 ## v2.3.0
 * set volume unit_number when cloning a vm (#151)
 * add unclustered esxi hosts back to cluster selection (#149)

data/CONTRIBUTING.md

@@ -1,18 +1,18 @@
-## Getting Involved
+# Getting Involved
 
 New contributors are always welcome, when it doubt please ask questions. We strive to be an open and welcoming community. Please be nice to one another.
 
-### Coding
+## Coding
 
-* Pick a task:
-* Offer feedback on open [pull requests](https://github.com/fog/fog-vsphere/pulls).
-* Review open [issues](https://github.com/fog/fog-vsphere/issues) for things to help on.
-* [Create an issue](https://github.com/fog/fog-vsphere/issues/new) to start a discussion on additions or features.
-* Fork the project, add your changes and tests to cover them in a topic branch.
-* Commit your changes and rebase against `fog/fog-vsphere` to ensure everything is up to date.
-* [Submit a pull request](https://github.com/fog/fog-vsphere/compare/)
+- Pick a task:
+- Offer feedback on open [pull requests](https://github.com/fog/fog-vsphere/pulls).
+- Review open [issues](https://github.com/fog/fog-vsphere/issues) for things to help on.
+- [Create an issue](https://github.com/fog/fog-vsphere/issues/new) to start a discussion on additions or features.
+- Fork the project, add your changes and tests to cover them in a topic branch.
+- Commit your changes and rebase against `fog/fog-vsphere` to ensure everything is up to date.
+- [Submit a pull request](https://github.com/fog/fog-vsphere/compare/)
 
-### Non-Coding
+## Non-Coding
 
-* Offer feedback on open [issues](https://github.com/fog/fog-vsphere/issues).
-* Organize or volunteer at events.
+- Offer feedback on open [issues](https://github.com/fog/fog-vsphere/issues).
+- Organize or volunteer at events.

data/README.md

@@ -1,11 +1,8 @@
 # Fog vSphere
+
 > VMware vSphere® provider for the Fog cloud services library
 
-[![Gem Version][gemfury-image]][gemfury-url]
-[![Build Status][travis-image]][travis-url]
-[![Dependency Status][gemnasium-image]][gemnasium-url]
-[![Test Coverage][coverage-image]][coverage-url]
-[![Code Climate][climate-image]][climate-url]
+[![Gem Version][gemfury-image]][gemfury-url] [![Build Status][travis-image]][travis-url] [![Test Coverage][coverage-image]][coverage-url] [![Code Climate][climate-image]][climate-url]
 
 The VMware vSphere® provider allows you to use the abstractions of the Fog cloud services library to communicate with vSphere.
 
@@ -91,7 +88,5 @@ To contribute to this project, add an issue or pull request. For more info on wh
 [coverage-url]: https://codeclimate.com/github/fog/fog-vsphere/coverage
 [gemfury-image]: https://badge.fury.io/rb/fog-vsphere.svg
 [gemfury-url]: http://badge.fury.io/rb/fog-vsphere
-[gemnasium-image]: https://gemnasium.com/fog/fog-vsphere.svg
-[gemnasium-url]: https://gemnasium.com/fog/fog-vsphere
 [travis-image]: https://travis-ci.org/fog/fog-vsphere.svg?branch=master
 [travis-url]: https://travis-ci.org/fog/fog-vsphere

data/lib/fog/vsphere/models/compute/server.rb

@@ -344,7 +344,14 @@ module Fog
 
         def initialize_volumes
           if attributes[:volumes] && attributes[:volumes].is_a?(Array)
-            attributes[:volumes].map! { |vol| vol.is_a?(Hash) ? service.volumes.new({ server: self }.merge(vol)) : vol }
+            attributes[:volumes].map! do |vol|
+              if vol.is_a?(Hash)
+                service.volumes.new({ server: self }.merge(vol))
+              else
+                vol.server = self
+                vol
+              end
+            end
           end
         end
 

data/lib/fog/vsphere/models/compute/volume.rb

@@ -98,6 +98,7 @@ module Fog
         end
 
         def set_unit_number
+          requires :server
           # When adding volumes to vsphere, if our unit_number is 7 or higher, vsphere will increment the unit_number
           # This is due to SCSI ID 7 being reserved for the pvscsi controller
           # When referring to a volume that already added using a unit_id of 7 or higher, we must refer to the actual SCSI ID

data/lib/fog/vsphere/requests/compute/create_vm.rb

@@ -31,7 +31,7 @@ module Fog
                    get_raw_host(attributes[:host], attributes[:cluster], attributes[:datacenter])
                  end
           # if any volume has a storage_pod set, we deploy the vm on a storage pod instead of the defined datastores
-          pod = get_storage_pod(attributes)
+          pod = get_storage_pod_from_volumes(attributes)
           vm = if pod
                  create_vm_on_storage_pod(pod, vm_cfg, vmFolder, resource_pool, attributes[:datacenter], host)
                else
@@ -73,7 +73,7 @@ module Fog
 
         # check if a storage pool is set on any of the volumes and return the first result found or nil
         # return early if vsphere revision is lower than 5 as this is not supported
-        def get_storage_pod(attributes)
+        def get_storage_pod_from_volumes(attributes)
           return unless @vsphere_rev.to_f >= 5
           volume = attributes[:volumes].detect { |volume| !(volume.storage_pod.nil? || volume.storage_pod.empty?) }
           volume.storage_pod if volume
@@ -83,7 +83,7 @@ module Fog
         # by default we prefer to keep it at the same place the (first) vmdk is located
         # if we deploy the vm on a storage pool, we have to return an empty string
         def vm_path_name(attributes)
-          return '' if get_storage_pod(attributes)
+          return '' if get_storage_pod_from_volumes(attributes)
           datastore = attributes[:volumes].first.datastore unless attributes[:volumes].empty?
           datastore ||= 'datastore1'
           "[#{datastore}]"
@@ -100,7 +100,7 @@ module Fog
           end
 
           if (disks = attributes[:volumes])
-            devices << disks.map { |disk| create_disk(disk, :add, storage_pod: get_storage_pod(attributes)) }
+            devices << disks.map { |disk| create_disk(disk, :add, storage_pod: get_storage_pod_from_volumes(attributes)) }
           end
 
           if (cdroms = attributes[:cdroms])
@@ -171,6 +171,12 @@ module Fog
                 switchUuid: raw_network.config.distributedVirtualSwitch.uuid
               )
             )
+          elsif raw_network.is_a? RbVmomi::VIM::OpaqueNetwork
+            RbVmomi::VIM.VirtualEthernetCardOpaqueNetworkBackingInfo(
+              opaqueNetworkType: raw_network.summary.opaqueNetworkType,
+              opaqueNetworkId: raw_network.summary.opaqueNetworkId
+            )
+
           else
             RbVmomi::VIM.VirtualEthernetCardNetworkBackingInfo(deviceName: nic.network)
           end

data/lib/fog/vsphere/requests/compute/get_cluster.rb

@@ -10,8 +10,12 @@ module Fog
 
         protected
 
-        def get_raw_cluster(name, datacenter_name)
-          dc = find_raw_datacenter(datacenter_name)
+        def get_raw_cluster(name, datacenter_name_or_obj)
+          dc = if datacenter_name_or_obj.is_a?(String)
+                 find_raw_datacenter(datacenter_name_or_obj)
+               else
+                 datacenter_name_or_obj
+               end
           dc.find_compute_resource(name)
         end
       end

data/lib/fog/vsphere/requests/compute/get_datastore.rb

@@ -3,9 +3,9 @@ module Fog
     class Vsphere
       class Real
         def get_datastore(name, datacenter_name)
-          datastore = get_raw_datastore(name, datacenter_name)
+          datastore = list_datastores(datacenter: datacenter_name).detect { |ds| ds[:name] == name }
           raise(Fog::Compute::Vsphere::NotFound) unless datastore
-          datastore_attributes(datastore, datacenter_name)
+          datastore
         end
 
         protected

data/lib/fog/vsphere/requests/compute/get_folder.rb

@@ -52,7 +52,7 @@ module Fog
             name: folder.name,
             parent: folder.parent.name,
             datacenter: datacenter_name,
-            type: folder_type(folder),
+            type: folder_type(folder.childType),
             path: folder_path(folder)
           }
         end
@@ -61,8 +61,7 @@ module Fog
           '/' + folder.path.map(&:last).join('/')
         end
 
-        def folder_type(folder)
-          types = folder.childType
+        def folder_type(types)
           return :vm        if types.include?('VirtualMachine')
           return :network   if types.include?('Network')
           return :datastore if types.include?('Datastore')

data/lib/fog/vsphere/requests/compute/get_storage_pod.rb

@@ -3,9 +3,9 @@ module Fog
     class Vsphere
       class Real
         def get_storage_pod(name, datacenter_name)
-          storage_pod = get_raw_storage_pod(name, datacenter_name)
+          storage_pod = list_storage_pods(datacenter: datacenter_name).detect { |pod| pod[:name] == name }
           raise(Fog::Compute::Vsphere::NotFound) unless storage_pod
-          storage_pod_attributes(storage_pod, datacenter_name)
+          storage_pod
         end
 
         protected
@@ -13,6 +13,10 @@ module Fog
         def get_raw_storage_pod(name, datacenter_name)
           raw_storage_pods(datacenter_name).detect { |pod| pod.name == name }
         end
+
+        def raw_storage_pods(datacenter_name)
+          list_container_view(datacenter_name, 'StoragePod')
+        end
       end
 
       class Mock

data/lib/fog/vsphere/requests/compute/list_datastores.rb

@@ -7,32 +7,76 @@ module Fog
           cluster_name = filters.fetch(:cluster, nil)
           # default to show all datastores
           only_active = filters[:accessible] || false
-          raw_datastores(datacenter_name, cluster_name).map do |datastore|
-            next if only_active && !datastore.summary.accessible
-            datastore_attributes(datastore, datacenter_name)
+
+          dc = find_raw_datacenter(datacenter_name)
+
+          datastores = if cluster_name
+                         cluster = get_raw_cluster(cluster_name, dc)
+                         property_collector_results(datastore_cluster_filter_spec(cluster))
+                       else
+                         property_collector_results(datastore_filter_spec(dc))
+                       end
+
+          datastores.map do |datastore|
+            next if only_active && !datastore['summary.accessible']
+            map_attrs_to_hash(datastore, datastore_attribute_mapping).merge(
+              datacenter: datacenter_name,
+              id: managed_obj_id(datastore.obj)
+            )
           end.compact
         end
 
-        def raw_datastores(datacenter_name, cluster = nil)
-          if cluster.nil?
-            find_raw_datacenter(datacenter_name).datastore
-          else
-            get_raw_cluster(cluster, datacenter_name).datastore
-          end
+        protected
+
+        def datastore_filter_spec(obj)
+          RbVmomi::VIM.PropertyFilterSpec(
+            objectSet: [
+              obj: obj.datastoreFolder,
+              skip: true,
+              selectSet: [
+                folder_traversal_spec
+              ]
+            ],
+            propSet: datastore_filter_prop_set
+          )
         end
 
-        protected
+        def datastore_cluster_filter_spec(obj)
+          RbVmomi::VIM.PropertyFilterSpec(
+            objectSet: [
+              obj: obj,
+              skip: true,
+              selectSet: [
+                compute_resource_datastore_traversal_spec
+              ]
+            ],
+            propSet: datastore_filter_prop_set
+          )
+        end
+
+        def datastore_filter_prop_set
+          [
+            { type: 'Datastore', pathSet: datastore_attribute_mapping.values }
+          ]
+        end
+
+        def compute_resource_datastore_traversal_spec
+          RbVmomi::VIM.TraversalSpec(
+            name: 'computeResourceDatastoreTraversalSpec',
+            type: 'ComputeResource',
+            path: 'datastore',
+            skip: false
+          )
+        end
 
-        def datastore_attributes(datastore, datacenter)
+        def datastore_attribute_mapping
           {
-            id: managed_obj_id(datastore),
-            name: datastore.name,
-            accessible: datastore.summary.accessible,
-            type: datastore.summary.type,
-            freespace: datastore.summary.freeSpace,
-            capacity: datastore.summary.capacity,
-            uncommitted: datastore.summary.uncommitted,
-            datacenter: datacenter
+            name: 'summary.name',
+            accessible: 'summary.accessible',
+            type: 'summary.type',
+            freespace: 'summary.freeSpace',
+            capacity: 'summary.capacity',
+            uncommitted: 'summary.uncommitted'
           }
         end
       end

data/lib/fog/vsphere/requests/compute/list_folders.rb

@@ -23,13 +23,130 @@ module Fog
         def list_folders(filters = {})
           path            = filters[:path] || filters['path'] || ''
           datacenter_name = filters[:datacenter]
-          get_raw_vmfolders(path, datacenter_name).map do |folder|
-            folder_attributes(folder, datacenter_name)
+
+          # if we don't need to display folders for a specific path
+          # we can easily use a property collector to get the
+          # data in an efficient manner from vsphere
+          # otherwise we use a much slower implementation
+          unless path.nil? || path.empty?
+            return get_raw_vmfolders(path, datacenter_name).map do |folder|
+              folder_attributes(folder, datacenter_name)
+            end
           end
+
+          root_folder = connection.serviceContent.rootFolder
+
+          results = property_collector_results(folder_filter_spec(root_folder))
+
+          folder_inventory = generate_folder_inventory(results)
+
+          folders = results.select { |result| result.obj.is_a?(RbVmomi::VIM::Folder) && result['childType'].include?('VirtualMachine') }
+
+          folders.map do |folder|
+            folder_id = managed_obj_id(folder.obj)
+            parent_id = folder['parent']._ref if folder['parent']
+            path = lookup_folder_path(folder_inventory, folder_id)
+            next unless path.include?(datacenter_name) # skip folders from another datacenter
+            map_attrs_to_hash(folder, folder_attribute_mapping).merge(
+              datacenter: datacenter_name,
+              parent: lookup_folder_name(folder_inventory, parent_id),
+              path: path.join('/'),
+              type: folder_type(folder['childType']),
+              id: folder_id
+            )
+          end.compact
         end
 
         protected
 
+        def folder_filter_spec(root_folder)
+          RbVmomi::VIM.PropertyFilterSpec(
+            objectSet: [
+              obj: root_folder,
+              skip: false,
+              selectSet: [
+                dc_folder_traversal_spec,
+                datacenter_to_vm_folder_traversal_spec
+              ]
+            ],
+            propSet: [
+              { type: 'Folder', pathSet: folder_attribute_mapping.values + %w[parent childType] },
+              { type: 'Datacenter', pathSet: %w[name parent] }
+            ]
+          )
+        end
+
+        def dc_folder_traversal_spec
+          RbVmomi::VIM.TraversalSpec(
+            name: 'dcFolderTraversalSpec',
+            type: 'Folder',
+            path: 'childEntity',
+            skip: false,
+            selectSet: [
+              RbVmomi::VIM.SelectionSpec(name: 'dcFolderTraversalSpec'),
+              RbVmomi::VIM.SelectionSpec(name: 'DatacenterToVmFolderTraversalSpec')
+            ]
+          )
+        end
+
+        def datacenter_to_vm_folder_traversal_spec
+          RbVmomi::VIM.TraversalSpec(
+            name: 'DatacenterToVmFolderTraversalSpec',
+            type: 'Datacenter',
+            path: 'vmFolder',
+            skip: false,
+            selectSet: [
+              RbVmomi::VIM.SelectionSpec(name: 'dcFolderTraversalSpec')
+            ]
+          )
+        end
+
+        def lookup_folder_name(inventory, folder_id)
+          folder = inventory[folder_id]
+          return '' unless folder
+          folder[:name]
+        end
+
+        def lookup_folder_path(inventory, folder_id)
+          folder = inventory[folder_id]
+          return '' unless folder
+          folder[:path]
+        end
+
+        def set_folder_paths(folder_inventory) # rubocop:disable Naming/AccessorMethodName
+          folder_inventory.each do |ref, props|
+            props[:path] = ['', lookup_parent_folders(folder_inventory, ref).reverse].flatten
+          end
+        end
+
+        def lookup_parent_folders(folder_inventory, ref)
+          return [] unless folder_inventory[ref]
+          return [folder_inventory[ref][:name]] if folder_inventory[ref][:parent].nil?
+          [folder_inventory[ref][:name], lookup_parent_folders(folder_inventory, folder_inventory[ref][:parent])].flatten
+        end
+
+        def folder_attribute_mapping
+          {
+            name: 'name'
+          }
+        end
+
+        def generate_folder_inventory(folders)
+          folder_inventory = folders.each_with_object({}) do |folder, inventory|
+            parent = if folder['parent'].nil?
+                       nil
+                     else
+                       folder['parent']._ref
+                     end
+            inventory[folder.obj._ref] = {
+              name: folder['name'],
+              parent: parent
+            }
+          end
+          set_folder_paths(folder_inventory)
+          folder_inventory
+        end
+
         def get_raw_vmfolders(path, datacenter_name)
           folder = get_raw_vmfolder(path, datacenter_name)
           child_folders(folder).flatten.compact

data/lib/fog/vsphere/requests/compute/list_storage_pods.rb

@@ -4,26 +4,45 @@ module Fog
       class Real
         def list_storage_pods(filters = {})
           datacenter_name = filters[:datacenter]
-          raw_storage_pods(datacenter_name).map do |storage_pod|
-            storage_pod_attributes(storage_pod, datacenter_name)
-          end.compact
-        end
 
-        private
+          dc = find_raw_datacenter(datacenter_name)
+
+          storage_pods = property_collector_results(storage_pod_filter_spec(dc))
 
-        def raw_storage_pods(datacenter_name)
-          list_container_view(datacenter_name, 'StoragePod')
+          storage_pods.map do |storage_pod|
+            map_attrs_to_hash(storage_pod, storage_pod_attribute_mapping).merge(
+              datacenter: datacenter_name,
+              id: managed_obj_id(storage_pod.obj)
+            )
+          end
         end
 
         protected
 
-        def storage_pod_attributes(storage_pod, datacenter)
+        def storage_pod_filter_spec(obj)
+          RbVmomi::VIM.PropertyFilterSpec(
+            objectSet: [
+              obj: obj.datastoreFolder,
+              skip: true,
+              selectSet: [
+                folder_traversal_spec
+              ]
+            ],
+            propSet: storage_pod_filter_prop_set
+          )
+        end
+
+        def storage_pod_filter_prop_set
+          [
+            { type: 'StoragePod', pathSet: storage_pod_attribute_mapping.values }
+          ]
+        end
+
+        def storage_pod_attribute_mapping
           {
-            id: managed_obj_id(storage_pod),
-            name: storage_pod.name,
-            freespace: storage_pod.summary.freeSpace,
-            capacity: storage_pod.summary.capacity,
-            datacenter: datacenter
+            name: 'name',
+            freespace: 'summary.freeSpace',
+            capacity: 'summary.capacity'
           }
         end
       end

data/lib/fog/vsphere/requests/compute/list_vm_interfaces.rb

@@ -61,11 +61,19 @@ module Fog
 
         private
 
+        # rubocop:disable Style/ConditionalAssignment
         def raw_to_hash(nic)
+          if nic.backing.respond_to?(:network)
+            network = nic.backing.network.name
+          elsif nic.backing.respond_to?(:port)
+            network = nic.backing.port.portgroupKey
+          else
+            network = nil
+          end
           {
             name: nic.deviceInfo.label,
             mac: nic.macAddress,
-            network: nic.backing.respond_to?('network') ? nic.backing.network.name : nic.backing.port.portgroupKey,
+            network: network,
             status: nic.connectable.status,
             connected: nic.connectable.connected,
             summary: nic.deviceInfo.summary,
@@ -73,6 +81,7 @@ module Fog
             key: nic.key
           }
         end
+        # rubocop:enable Style/ConditionalAssignment
       end
 
       class Mock

data/lib/fog/vsphere/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module Vsphere
-    VERSION = '2.3.0'.freeze
+    VERSION = '2.4.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fog-vsphere
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.4.0
 platform: ruby
 authors:
 - J.R. Garcia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-09 00:00:00.000000000 Z
+date: 2018-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog-core