vmpooler 0.12.0 → 0.14.0

data/lib/vmpooler/providers/base.rb

@@ -14,10 +14,11 @@ module Vmpooler
         # Provider options passed in during initialization
         attr_reader :provider_options
 
-        def initialize(config, logger, metrics, name, options)
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
           @config = config
           @logger = logger
           @metrics = metrics
+          @redis = redis_connection_pool
           @provider_name = name
 
           # Ensure that there is not a nil provider configuration

data/lib/vmpooler/providers/dummy.rb

@@ -9,8 +9,8 @@ module Vmpooler
       class Dummy < Vmpooler::PoolManager::Provider::Base
         # Fake VM Provider for testing
 
-        def initialize(config, logger, metrics, name, options)
-          super(config, logger, metrics, name, options)
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
+          super(config, logger, metrics, redis_connection_pool, name, options)
           dummyfilename = provider_config['filename']
 
           # This initial_state option is only intended to be used by spec tests
@@ -29,7 +29,8 @@ module Vmpooler
           logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}")
           @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new(
             metrics: metrics,
-            metric_prefix: "#{name}_provider_connection_pool",
+            connpool_type: 'provider_connection_pool',
+            connpool_provider: name,
             size: connpool_size,
             timeout: connpool_timeout
           ) do

data/lib/vmpooler/providers/vsphere.rb

@@ -9,8 +9,8 @@ module Vmpooler
         # The connection_pool method is normally used only for testing
         attr_reader :connection_pool
 
-        def initialize(config, logger, metrics, name, options)
-          super(config, logger, metrics, name, options)
+        def initialize(config, logger, metrics, redis_connection_pool, name, options)
+          super(config, logger, metrics, redis_connection_pool, name, options)
 
           task_limit = global_config[:config].nil? || global_config[:config]['task_limit'].nil? ? 10 : global_config[:config]['task_limit'].to_i
           # The default connection pool size is:
@@ -25,7 +25,8 @@ module Vmpooler
           logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}")
           @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new(
             metrics: metrics,
-            metric_prefix: "#{name}_provider_connection_pool",
+            connpool_type: 'provider_connection_pool',
+            connpool_provider: name,
             size: connpool_size,
             timeout: connpool_timeout
           ) do
@@ -39,6 +40,7 @@ module Vmpooler
           end
           @provider_hosts = {}
           @provider_hosts_lock = Mutex.new
+          @redis = redis_connection_pool
         end
 
         # name of the provider class
@@ -59,12 +61,16 @@ module Vmpooler
         def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl)
           try = 0 if try.nil?
           max_tries = 3
-          $redis.srem("vmpooler__completed__#{pool}", vm_name)
-          $redis.hdel("vmpooler__active__#{pool}", vm_name)
-          $redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now)
-
-          # Auto-expire metadata key
-          $redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60))
+          @redis.with_metrics do |redis|
+            redis.multi
+            redis.srem("vmpooler__completed__#{pool}", vm_name)
+            redis.hdel("vmpooler__active__#{pool}", vm_name)
+            redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now)
+
+            # Auto-expire metadata key
+            redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60))
+            redis.exec
+          end
 
           start = Time.now
 
@@ -293,7 +299,6 @@ module Vmpooler
             template_path = pool['template']
             target_folder_path = pool['folder']
             target_datastore = pool['datastore']
-            target_cluster_name = get_target_cluster_from_config(pool_name)
             target_datacenter_name = get_target_datacenter_from_config(pool_name)
 
             # Get the template VM object
@@ -315,35 +320,23 @@ module Vmpooler
               ]
             )
 
-            # Put the VM in the specified folder and resource pool
-            relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec(
-              datastore: find_datastore(target_datastore, connection, target_datacenter_name),
-              diskMoveType: get_disk_backing(pool)
-            )
-
-            manage_host_selection = @config[:config]['manage_host_selection'] if @config[:config].key?('manage_host_selection')
-            if manage_host_selection
-              run_select_hosts(pool_name, @provider_hosts)
-              target_host = select_next_host(pool_name, @provider_hosts)
-              host_object = find_host_by_dnsname(connection, target_host)
-              relocate_spec.host = host_object
-            else
-              # Choose a cluster/host to place the new VM on
-              target_cluster_object = find_cluster(target_cluster_name, connection, target_datacenter_name)
-              relocate_spec.pool = target_cluster_object.resourcePool
+            # Check if alternate network configuration is specified and add configuration
+            if pool.key?('network')
+              template_vm_network_device = template_vm_object.config.hardware.device.grep(RbVmomi::VIM::VirtualEthernetCard).first
+              network_name = pool['network']
+              network_device = set_network_device(target_datacenter_name, template_vm_network_device, network_name, connection)
+              config_spec.deviceChange = [{ operation: 'edit', device: network_device }]
             end
 
+            # Put the VM in the specified folder and resource pool
+            relocate_spec = create_relocate_spec(target_datastore, target_datacenter_name, pool_name, connection)
+
             # Create a clone spec
-            clone_spec = RbVmomi::VIM.VirtualMachineCloneSpec(
-              location: relocate_spec,
-              config: config_spec,
-              powerOn: true,
-              template: false
-            )
+            clone_spec = create_clone_spec(relocate_spec, config_spec)
 
             begin
               vm_target_folder = find_vm_folder(pool_name, connection)
-              vm_target_folder = create_folder(connection, target_folder_path, target_datacenter_name) if vm_target_folder.nil? && @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
+              vm_target_folder ||= create_folder(connection, target_folder_path, target_datacenter_name) if @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
             rescue StandardError
               if @config[:config].key?('create_folders') && (@config[:config]['create_folders'] == true)
                 vm_target_folder = create_folder(connection, target_folder_path, target_datacenter_name)
@@ -351,6 +344,7 @@ module Vmpooler
                 raise
               end
             end
+            raise ArgumentError, "Cannot find the configured folder for #{pool_name} #{target_folder_path}" unless vm_target_folder
 
             # Create the new VM
             new_vm_object = template_vm_object.CloneVM_Task(
@@ -364,6 +358,81 @@ module Vmpooler
           vm_hash
         end
 
+        def create_relocate_spec(target_datastore, target_datacenter_name, pool_name, connection)
+          pool = pool_config(pool_name)
+          target_cluster_name = get_target_cluster_from_config(pool_name)
+
+          relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec(
+            datastore: find_datastore(target_datastore, connection, target_datacenter_name),
+            diskMoveType: get_disk_backing(pool)
+          )
+          manage_host_selection = @config[:config]['manage_host_selection'] if @config[:config].key?('manage_host_selection')
+          if manage_host_selection
+            run_select_hosts(pool_name, @provider_hosts)
+            target_host = select_next_host(pool_name, @provider_hosts)
+            host_object = find_host_by_dnsname(connection, target_host)
+            relocate_spec.host = host_object
+          else
+            # Choose a cluster/host to place the new VM on
+            target_cluster_object = find_cluster(target_cluster_name, connection, target_datacenter_name)
+            relocate_spec.pool = target_cluster_object.resourcePool
+          end
+          relocate_spec
+        end
+
+        def create_clone_spec(relocate_spec, config_spec)
+          RbVmomi::VIM.VirtualMachineCloneSpec(
+            location: relocate_spec,
+            config: config_spec,
+            powerOn: true,
+            template: false
+          )
+        end
+
+        def set_network_device(datacenter_name, template_vm_network_device, network_name, connection)
+          # Retrieve network object
+          datacenter = connection.serviceInstance.find_datacenter(datacenter_name)
+          new_network = datacenter.network.find { |n| n.name == network_name }
+
+          raise("Cannot find network #{network_name} in datacenter #{datacenter_name}") unless new_network
+
+          # Determine network device type
+          # All possible device type options here: https://vdc-download.vmware.com/vmwb-repository/dcr-public/98d63b35-d822-47fe-a87a-ddefd469df06/2e3c7b58-f2bd-486e-8bb1-a75eb0640bee/doc/vim.vm.device.VirtualEthernetCard.html
+          network_device =
+            if template_vm_network_device.is_a? RbVmomi::VIM::VirtualVmxnet2
+              RbVmomi::VIM.VirtualVmxnet2
+            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualVmxnet3
+              RbVmomi::VIM.VirtualVmxnet3
+            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualE1000
+              RbVmomi::VIM.VirtualE1000
+            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualE1000e
+              RbVmomi::VIM.VirtualE1000e
+            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualSriovEthernetCard
+              RbVmomi::VIM.VirtualSriovEthernetCard
+            else
+              RbVmomi::VIM.VirtualPCNet32
+            end
+
+          # Set up new network device attributes
+          network_device.key = template_vm_network_device.key
+          network_device.deviceInfo = RbVmomi::VIM.Description(
+            label: template_vm_network_device.deviceInfo.label,
+            summary: network_name
+          )
+          network_device.backing = RbVmomi::VIM.VirtualEthernetCardNetworkBackingInfo(
+            deviceName: network_name,
+            network: new_network,
+            useAutoDetect: false
+          )
+          network_device.addressType = 'assigned'
+          network_device.connectable = RbVmomi::VIM.VirtualDeviceConnectInfo(
+            allowGuestControl: true,
+            startConnected: true,
+            connected: true
+          )
+          network_device
+        end
+
         def create_disk(pool_name, vm_name, disk_size)
           pool = pool_config(pool_name)
           raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?
@@ -968,22 +1037,24 @@ module Vmpooler
             begin
               connection = ensured_vsphere_connection(pool_object)
               vm_hash = get_vm_details(pool_name, vm_name, connection)
-              $redis.hset("vmpooler__vm__#{vm_name}", 'host', vm_hash['host_name'])
-              migration_limit = @config[:config]['migration_limit'] if @config[:config].key?('migration_limit')
-              migration_count = $redis.scard('vmpooler__migration')
-              if migration_enabled? @config
-                if migration_count >= migration_limit
-                  logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}. No migration will be evaluated since the migration_limit has been reached")
-                  break
-                end
-                run_select_hosts(pool_name, @provider_hosts)
-                if vm_in_target?(pool_name, vm_hash['host_name'], vm_hash['architecture'], @provider_hosts)
-                  logger.log('s', "[ ] [#{pool_name}] No migration required for '#{vm_name}' running on #{vm_hash['host_name']}")
+              @redis.with_metrics do |redis|
+                redis.hset("vmpooler__vm__#{vm_name}", 'host', vm_hash['host_name'])
+                migration_count = redis.scard('vmpooler__migration')
+                migration_limit = @config[:config]['migration_limit'] if @config[:config].key?('migration_limit')
+                if migration_enabled? @config
+                  if migration_count >= migration_limit
+                    logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}. No migration will be evaluated since the migration_limit has been reached")
+                    break
+                  end
+                  run_select_hosts(pool_name, @provider_hosts)
+                  if vm_in_target?(pool_name, vm_hash['host_name'], vm_hash['architecture'], @provider_hosts)
+                    logger.log('s', "[ ] [#{pool_name}] No migration required for '#{vm_name}' running on #{vm_hash['host_name']}")
+                  else
+                    migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
+                  end
                 else
-                  migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
+                  logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
                 end
-              else
-                logger.log('s', "[ ] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
               end
             rescue StandardError
               logger.log('s', "[!] [#{pool_name}] '#{vm_name}' is running on #{vm_hash['host_name']}")
@@ -993,15 +1064,23 @@ module Vmpooler
         end
 
         def migrate_vm_to_new_host(pool_name, vm_name, vm_hash, connection)
-          $redis.sadd('vmpooler__migration', vm_name)
+          @redis.with_metrics do |redis|
+            redis.sadd('vmpooler__migration', vm_name)
+          end
           target_host_name = select_next_host(pool_name, @provider_hosts, vm_hash['architecture'])
           target_host_object = find_host_by_dnsname(connection, target_host_name)
           finish = migrate_vm_and_record_timing(pool_name, vm_name, vm_hash, target_host_object, target_host_name)
-          $redis.hset("vmpooler__vm__#{vm_name}", 'host', target_host_name)
-          $redis.hset("vmpooler__vm__#{vm_name}", 'migrated', true)
+          @redis.with_metrics do |redis|
+            redis.multi
+            redis.hset("vmpooler__vm__#{vm_name}", 'host', target_host_name)
+            redis.hset("vmpooler__vm__#{vm_name}", 'migrated', true)
+            redis.exec
+          end
           logger.log('s', "[>] [#{pool_name}] '#{vm_name}' migrated from #{vm_hash['host_name']} to #{target_host_name} in #{finish} seconds")
         ensure
-          $redis.srem('vmpooler__migration', vm_name)
+          @redis.with_metrics do |redis|
+            redis.srem('vmpooler__migration', vm_name)
+          end
         end
 
         def migrate_vm_and_record_timing(pool_name, vm_name, vm_hash, target_host_object, dest_host_name)
@@ -1011,9 +1090,13 @@ module Vmpooler
           metrics.timing("migrate.#{pool_name}", finish)
           metrics.increment("migrate_from.#{vm_hash['host_name']}")
           metrics.increment("migrate_to.#{dest_host_name}")
-          checkout_to_migration = format('%<time>.2f', time: Time.now - Time.parse($redis.hget("vmpooler__vm__#{vm_name}", 'checkout')))
-          $redis.hset("vmpooler__vm__#{vm_name}", 'migration_time', finish)
-          $redis.hset("vmpooler__vm__#{vm_name}", 'checkout_to_migration', checkout_to_migration)
+          @redis.with_metrics do |redis|
+            checkout_to_migration = format('%<time>.2f', time: Time.now - Time.parse(redis.hget("vmpooler__vm__#{vm_name}", 'checkout')))
+            redis.multi
+            redis.hset("vmpooler__vm__#{vm_name}", 'migration_time', finish)
+            redis.hset("vmpooler__vm__#{vm_name}", 'checkout_to_migration', checkout_to_migration)
+            redis.exec
+          end
           finish
         end
 

data/lib/vmpooler/util/parsing.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# utility class shared between apps
+module Vmpooler
+  class Parsing
+    def self.get_platform_pool_count(requested, &_block)
+      requested_platforms = requested.split(',')
+      requested_platforms.each do |platform|
+        platform_alias, pool, count = platform.split(':')
+        raise ArgumentError if platform_alias.nil? || pool.nil? || count.nil?
+
+        yield platform_alias, pool, count
+      end
+    end
+  end
+end

data/lib/vmpooler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vmpooler
-  VERSION = '0.12.0'
+  VERSION = '0.14.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vmpooler
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-28 00:00:00.000000000 Z
+date: 2020-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pickup
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: net-ldap
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +164,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -191,13 +219,18 @@ files:
 - lib/vmpooler/api.rb
 - lib/vmpooler/api/dashboard.rb
 - lib/vmpooler/api/helpers.rb
+- lib/vmpooler/api/request_logger.rb
 - lib/vmpooler/api/reroute.rb
 - lib/vmpooler/api/v1.rb
 - lib/vmpooler/dashboard.rb
-- lib/vmpooler/dummy_statsd.rb
 - lib/vmpooler/generic_connection_pool.rb
-- lib/vmpooler/graphite.rb
 - lib/vmpooler/logger.rb
+- lib/vmpooler/metrics.rb
+- lib/vmpooler/metrics/dummy_statsd.rb
+- lib/vmpooler/metrics/graphite.rb
+- lib/vmpooler/metrics/promstats.rb
+- lib/vmpooler/metrics/promstats/collector_middleware.rb
+- lib/vmpooler/metrics/statsd.rb
 - lib/vmpooler/pool_manager.rb
 - lib/vmpooler/providers.rb
 - lib/vmpooler/providers/base.rb
@@ -214,7 +247,7 @@ files:
 - lib/vmpooler/public/lib/dashboard.js
 - lib/vmpooler/public/lib/jquery.min.js
 - lib/vmpooler/public/vmpooler.css
-- lib/vmpooler/statsd.rb
+- lib/vmpooler/util/parsing.rb
 - lib/vmpooler/version.rb
 - lib/vmpooler/views/dashboard.erb
 - lib/vmpooler/views/layout.erb
@@ -237,7 +270,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: vmpooler provides configurable pools of instantly-available (running) virtual

data/lib/vmpooler/dummy_statsd.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Vmpooler
-  class DummyStatsd
-    attr_reader :server, :port, :prefix
-
-    def initialize(*)
-    end
-
-    def increment(*)
-      true
-    end
-
-    def gauge(*)
-      true
-    end
-
-    def timing(*)
-      true
-    end
-  end
-end