vmpooler 0.13.1 → 0.14.2

data/lib/vmpooler/metrics/promstats/collector_middleware.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+# This is an adapted Collector module for vmpooler based on the sample implementation
+# available in the prometheus client_ruby library
+# https://github.com/prometheus/client_ruby/blob/master/lib/prometheus/middleware/collector.rb
+#
+# The code was also failing Rubocop on PR check, so have addressed all the offenses.
+#
+# The method strip_hostnames_from_path (originally strip_ids_from_path) has been adapted
+# to add a match for hostnames in paths # to replace with a single ":hostname" string to
+# avoid # proliferation of stat lines for # each new vm hostname deleted, modified or
+# otherwise queried.
+
+require 'benchmark'
+require 'prometheus/client'
+require 'vmpooler/logger'
+
+module Vmpooler
+  class Metrics
+    class Promstats
+      # CollectorMiddleware is an implementation of Rack Middleware customised
+      # for vmpooler use.
+      #
+      # By default metrics are registered on the global registry. Set the
+      # `:registry` option to use a custom registry.
+      #
+      # By default metrics all have the prefix "http_server". Set to something
+      # else if you like.
+      #
+      # The request counter metric is broken down by code, method and path by
+      # default. Set the `:counter_label_builder` option to use a custom label
+      # builder.
+      #
+      # The request duration metric is broken down by method and path by default.
+      # Set the `:duration_label_builder` option to use a custom label builder.
+      #
+      # Label Builder functions will receive a Rack env and a status code, and must
+      # return a hash with the labels for that request. They must also accept an empty
+      # env, and return a hash with the correct keys. This is necessary to initialize
+      # the metrics with the correct set of labels.
+      class CollectorMiddleware
+        attr_reader :app, :registry
+
+        def initialize(app, options = {})
+          @app = app
+          @registry = options[:registry] || Prometheus::Client.registry
+          @metrics_prefix = options[:metrics_prefix] || 'http_server'
+
+          init_request_metrics
+          init_exception_metrics
+        end
+
+        def call(env) # :nodoc:
+          trace(env) { @app.call(env) }
+        end
+
+        protected
+
+        def init_request_metrics
+          @requests = @registry.counter(
+            :"#{@metrics_prefix}_requests_total",
+            docstring:
+              'The total number of HTTP requests handled by the Rack application.',
+            labels: %i[code method path]
+          )
+          @durations = @registry.histogram(
+            :"#{@metrics_prefix}_request_duration_seconds",
+            docstring: 'The HTTP response duration of the Rack application.',
+            labels: %i[method path]
+          )
+        end
+
+        def init_exception_metrics
+          @exceptions = @registry.counter(
+            :"#{@metrics_prefix}_exceptions_total",
+            docstring: 'The total number of exceptions raised by the Rack application.',
+            labels: [:exception]
+          )
+        end
+
+        def trace(env)
+          response = nil
+          duration = Benchmark.realtime { response = yield }
+          record(env, response.first.to_s, duration)
+          response
+        rescue StandardError => e
+          @exceptions.increment(labels: { exception: e.class.name })
+          raise
+        end
+
+        def record(env, code, duration)
+          counter_labels = {
+            code: code,
+            method: env['REQUEST_METHOD'].downcase,
+            path: strip_hostnames_from_path(env['PATH_INFO'])
+          }
+
+          duration_labels = {
+            method: env['REQUEST_METHOD'].downcase,
+            path: strip_hostnames_from_path(env['PATH_INFO'])
+          }
+
+          @requests.increment(labels: counter_labels)
+          @durations.observe(duration, labels: duration_labels)
+        rescue # rubocop:disable Style/RescueStandardError
+          nil
+        end
+
+        def strip_hostnames_from_path(path)
+          # Custom for /vm path - so we just collect aggrate stats for all usage along this one
+          # path. Custom counters are then added more specific endpoints in v1.rb
+          # Since we aren't parsing UID/GIDs as in the original example, these are removed.
+          # Similarly, request IDs are also stripped from the /ondemand path.
+          path
+            .gsub(%r{/vm/.+$}, '/vm')
+            .gsub(%r{/ondemand/.+$}, '/ondemand')
+        end
+      end
+    end
+  end
+end

data/lib/vmpooler/metrics/statsd.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'rubygems' unless defined?(Gem)
+require 'statsd'
+
+module Vmpooler
+  class Metrics
+    class Statsd < Metrics
+      attr_reader :server, :port, :prefix
+
+      def initialize(logger, params = {})
+        raise ArgumentError, "Statsd server is required. Config: #{params.inspect}" if params['server'].nil? || params['server'].empty?
+
+        host    = params['server']
+        @port   = params['port'] || 8125
+        @prefix = params['prefix'] || 'vmpooler'
+        @server = ::Statsd.new(host, @port)
+        @logger = logger
+      end
+
+      def increment(label)
+        server.increment(prefix + '.' + label)
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure incrementing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+
+      def gauge(label, value)
+        server.gauge(prefix + '.' + label, value)
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure updating gauge #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+
+      def timing(label, duration)
+        server.timing(prefix + '.' + label, duration)
+      rescue StandardError => e
+        @logger.log('s', "[!] Failure updating timing #{prefix}.#{label} on statsd server [#{server}:#{port}]: #{e}")
+      end
+    end
+  end
+end

data/lib/vmpooler/pool_manager.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'vmpooler/providers'
+require 'vmpooler/util/parsing'
 require 'spicy-proton'
 require 'resolv' # ruby standard lib
 
@@ -149,8 +150,11 @@ module Vmpooler
         redis.pipelined do
           redis.hset("vmpooler__active__#{pool}", vm, Time.now)
           redis.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
-          redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token']) if ondemandrequest_hash['token:token']
-          redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user']) if ondemandrequest_hash['token:user']
+          if ondemandrequest_hash['token:token']
+            redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token'])
+            redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user'])
+            redis.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i)
+          end
           redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
         end
         move_vm_queue(pool, vm, 'pending', 'running', redis)
@@ -490,15 +494,21 @@ module Vmpooler
       return if checkout.nil?
 
       user ||= 'unauthenticated'
-      unless jenkins_build_url
-        user = user.gsub('.', '_')
-        $metrics.increment("usage.#{user}.#{poolname}")
+      user = user.gsub('.', '_')
+      $metrics.increment("user.#{user}.#{poolname}")
+
+      return unless jenkins_build_url
+
+      if jenkins_build_url.include? 'litmus'
+        # Very simple filter for Litmus jobs - just count them coming through for the moment.
+        $metrics.increment("usage_litmus.#{user}.#{poolname}")
         return
       end
 
       url_parts = jenkins_build_url.split('/')[2..-1]
-      instance = url_parts[0]
+      jenkins_instance = url_parts[0].gsub('.', '_')
       value_stream_parts = url_parts[2].split('_')
+      value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') }
       value_stream = value_stream_parts.shift
       branch = value_stream_parts.pop
       project = value_stream_parts.shift
@@ -506,22 +516,9 @@ module Vmpooler
       build_metadata_parts = url_parts[3]
       component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts)
 
-      metric_parts = [
-        'usage',
-        user,
-        instance,
-        value_stream,
-        branch,
-        project,
-        job_name,
-        component_to_test,
-        poolname
-      ]
-
-      metric_parts = metric_parts.reject(&:nil?)
-      metric_parts = metric_parts.map { |s| s.gsub('.', '_') }
-
-      $metrics.increment(metric_parts.join('.'))
+      $metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{poolname}")
+      $metrics.increment("usage_branch_project.#{branch}.#{project}.#{poolname}")
+      $metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{poolname}")
     rescue StandardError => e
       $logger.log('d', "[!] [#{poolname}] failed while evaluating usage labels on '#{vm}' with an error: #{e}")
       raise
@@ -536,21 +533,20 @@ module Vmpooler
         next if value.nil?
         return value if key == match
       end
-      nil
+      'none'
     end
 
     def purge_unused_vms_and_folders
       global_purge = $config[:config]['purge_unconfigured_folders']
       providers = $config[:providers].keys
-      providers.each do |provider|
-        provider_purge = $config[:providers][provider]['purge_unconfigured_folders']
-        provider_purge = global_purge if provider_purge.nil?
+      providers.each do |provider_key|
+        provider_purge = $config[:providers][provider_key]['purge_unconfigured_folders'] || global_purge
         if provider_purge
           Thread.new do
             begin
-              purge_vms_and_folders($providers[provider.to_s])
+              purge_vms_and_folders(provider_key)
             rescue StandardError => e
-              $logger.log('s', "[!] failed while purging provider #{provider} VMs and folders with an error: #{e}")
+              $logger.log('s', "[!] failed while purging provider #{provider_key} VMs and folders with an error: #{e}")
             end
           end
         end
@@ -559,14 +555,13 @@ module Vmpooler
     end
 
     # Return a list of pool folders
-    def pool_folders(provider)
-      provider_name = provider.name
+    def pool_folders(provider_name)
       folders = {}
       $config[:pools].each do |pool|
-        next unless pool['provider'] == provider_name
+        next unless pool['provider'] == provider_name.to_s
 
         folder_parts = pool['folder'].split('/')
-        datacenter = provider.get_target_datacenter_from_config(pool['name'])
+        datacenter = $providers[provider_name.to_s].get_target_datacenter_from_config(pool['name'])
         folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
       end
       folders
@@ -580,8 +575,9 @@ module Vmpooler
       base.uniq
     end
 
-    def purge_vms_and_folders(provider)
-      configured_folders = pool_folders(provider)
+    def purge_vms_and_folders(provider_name)
+      provider = $providers[provider_name.to_s]
+      configured_folders = pool_folders(provider_name)
       base_folders = get_base_folders(configured_folders)
       whitelist = provider.provider_config['folder_whitelist']
       provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
@@ -1500,9 +1496,7 @@ module Vmpooler
     def vms_ready?(request_id, redis)
       catch :request_not_ready do
         request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
-        requested_platforms = request_hash['requested'].split(',')
-        requested_platforms.each do |platform|
-          platform_alias, pool, count = platform.split(':')
+        Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count|
           pools_filled = redis.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
           throw :request_not_ready unless pools_filled.to_i == count.to_i
         end
@@ -1554,9 +1548,7 @@ module Vmpooler
 
     def remove_vms_for_failed_request(request_id, expiration_ttl, redis)
       request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
-      requested_platforms = request_hash['requested'].split(',')
-      requested_platforms.each do |platform|
-        platform_alias, pool, _count = platform.split(':')
+      Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
         pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
         redis.pipelined do
           pools_filled&.each do |vm|

data/lib/vmpooler/providers/dummy.rb

@@ -29,7 +29,8 @@ module Vmpooler
           logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}")
           @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new(
             metrics: metrics,
-            metric_prefix: "#{name}_provider_connection_pool",
+            connpool_type: 'provider_connection_pool',
+            connpool_provider: name,
             size: connpool_size,
             timeout: connpool_timeout
           ) do

data/lib/vmpooler/providers/vsphere.rb

@@ -25,7 +25,8 @@ module Vmpooler
           logger.log('d', "[#{name}] ConnPool - Creating a connection pool of size #{connpool_size} with timeout #{connpool_timeout}")
           @connection_pool = Vmpooler::PoolManager::GenericConnectionPool.new(
             metrics: metrics,
-            metric_prefix: "#{name}_provider_connection_pool",
+            connpool_type: 'provider_connection_pool',
+            connpool_provider: name,
             size: connpool_size,
             timeout: connpool_timeout
           ) do
@@ -298,7 +299,6 @@ module Vmpooler
             template_path = pool['template']
             target_folder_path = pool['folder']
             target_datastore = pool['datastore']
-            target_cluster_name = get_target_cluster_from_config(pool_name)
             target_datacenter_name = get_target_datacenter_from_config(pool_name)
 
             # Get the template VM object
@@ -320,31 +320,19 @@ module Vmpooler
               ]
             )
 
-            # Put the VM in the specified folder and resource pool
-            relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec(
-              datastore: find_datastore(target_datastore, connection, target_datacenter_name),
-              diskMoveType: get_disk_backing(pool)
-            )
-
-            manage_host_selection = @config[:config]['manage_host_selection'] if @config[:config].key?('manage_host_selection')
-            if manage_host_selection
-              run_select_hosts(pool_name, @provider_hosts)
-              target_host = select_next_host(pool_name, @provider_hosts)
-              host_object = find_host_by_dnsname(connection, target_host)
-              relocate_spec.host = host_object
-            else
-              # Choose a cluster/host to place the new VM on
-              target_cluster_object = find_cluster(target_cluster_name, connection, target_datacenter_name)
-              relocate_spec.pool = target_cluster_object.resourcePool
+            # Check if alternate network configuration is specified and add configuration
+            if pool.key?('network')
+              template_vm_network_device = template_vm_object.config.hardware.device.grep(RbVmomi::VIM::VirtualEthernetCard).first
+              network_name = pool['network']
+              network_device = set_network_device(target_datacenter_name, template_vm_network_device, network_name, connection)
+              config_spec.deviceChange = [{ operation: 'edit', device: network_device }]
             end
 
+            # Put the VM in the specified folder and resource pool
+            relocate_spec = create_relocate_spec(target_datastore, target_datacenter_name, pool_name, connection)
+
             # Create a clone spec
-            clone_spec = RbVmomi::VIM.VirtualMachineCloneSpec(
-              location: relocate_spec,
-              config: config_spec,
-              powerOn: true,
-              template: false
-            )
+            clone_spec = create_clone_spec(relocate_spec, config_spec)
 
             begin
               vm_target_folder = find_vm_folder(pool_name, connection)
@@ -356,7 +344,7 @@ module Vmpooler
                 raise
               end
             end
-            raise ArgumentError, "Can not find the configured folder for #{pool_name} #{target_folder_path}" unless vm_target_folder
+            raise ArgumentError, "Cannot find the configured folder for #{pool_name} #{target_folder_path}" unless vm_target_folder
 
             # Create the new VM
             new_vm_object = template_vm_object.CloneVM_Task(
@@ -370,6 +358,81 @@ module Vmpooler
           vm_hash
         end
 
+        def create_relocate_spec(target_datastore, target_datacenter_name, pool_name, connection)
+          pool = pool_config(pool_name)
+          target_cluster_name = get_target_cluster_from_config(pool_name)
+
+          relocate_spec = RbVmomi::VIM.VirtualMachineRelocateSpec(
+            datastore: find_datastore(target_datastore, connection, target_datacenter_name),
+            diskMoveType: get_disk_backing(pool)
+          )
+          manage_host_selection = @config[:config]['manage_host_selection'] if @config[:config].key?('manage_host_selection')
+          if manage_host_selection
+            run_select_hosts(pool_name, @provider_hosts)
+            target_host = select_next_host(pool_name, @provider_hosts)
+            host_object = find_host_by_dnsname(connection, target_host)
+            relocate_spec.host = host_object
+          else
+            # Choose a cluster/host to place the new VM on
+            target_cluster_object = find_cluster(target_cluster_name, connection, target_datacenter_name)
+            relocate_spec.pool = target_cluster_object.resourcePool
+          end
+          relocate_spec
+        end
+
+        def create_clone_spec(relocate_spec, config_spec)
+          RbVmomi::VIM.VirtualMachineCloneSpec(
+            location: relocate_spec,
+            config: config_spec,
+            powerOn: true,
+            template: false
+          )
+        end
+
+        def set_network_device(datacenter_name, template_vm_network_device, network_name, connection)
+          # Retrieve network object
+          datacenter = connection.serviceInstance.find_datacenter(datacenter_name)
+          new_network = datacenter.network.find { |n| n.name == network_name }
+
+          raise("Cannot find network #{network_name} in datacenter #{datacenter_name}") unless new_network
+
+          # Determine network device type
+          # All possible device type options here: https://vdc-download.vmware.com/vmwb-repository/dcr-public/98d63b35-d822-47fe-a87a-ddefd469df06/2e3c7b58-f2bd-486e-8bb1-a75eb0640bee/doc/vim.vm.device.VirtualEthernetCard.html
+          network_device =
+            if template_vm_network_device.is_a? RbVmomi::VIM::VirtualVmxnet2
+              RbVmomi::VIM.VirtualVmxnet2
+            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualVmxnet3
+              RbVmomi::VIM.VirtualVmxnet3
+            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualE1000
+              RbVmomi::VIM.VirtualE1000
+            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualE1000e
+              RbVmomi::VIM.VirtualE1000e
+            elsif template_vm_network_device.is_a? RbVmomi::VIM::VirtualSriovEthernetCard
+              RbVmomi::VIM.VirtualSriovEthernetCard
+            else
+              RbVmomi::VIM.VirtualPCNet32
+            end
+
+          # Set up new network device attributes
+          network_device.key = template_vm_network_device.key
+          network_device.deviceInfo = RbVmomi::VIM.Description(
+            label: template_vm_network_device.deviceInfo.label,
+            summary: network_name
+          )
+          network_device.backing = RbVmomi::VIM.VirtualEthernetCardNetworkBackingInfo(
+            deviceName: network_name,
+            network: new_network,
+            useAutoDetect: false
+          )
+          network_device.addressType = 'assigned'
+          network_device.connectable = RbVmomi::VIM.VirtualDeviceConnectInfo(
+            allowGuestControl: true,
+            startConnected: true,
+            connected: true
+          )
+          network_device
+        end
+
         def create_disk(pool_name, vm_name, disk_size)
           pool = pool_config(pool_name)
           raise("Pool #{pool_name} does not exist for the provider #{name}") if pool.nil?