vmpooler 2.1.0 → 2.4.0

data/lib/vmpooler/pool_manager.rb

@@ -60,6 +60,7 @@ module Vmpooler
             to_set[k] = pool[k]
           end
           to_set['alias'] = pool['alias'].join(',') if to_set.key?('alias')
+          to_set['domain'] = Parsing.get_domain_for_pool(config, pool['name'])
           redis.hmset("vmpooler__pool__#{pool['name']}", to_set.to_a.flatten) unless to_set.empty?
         end
         previously_configured_pools.each do |pool|
@@ -118,7 +119,13 @@ module Vmpooler
           pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
           redis.multi
           redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
-          redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}") if request_id
+          if request_id
+            ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+            if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
+              # will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
+              redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
+            end
+          end
           redis.exec
           $metrics.increment("errors.markedasfailed.#{pool}")
           $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
@@ -148,15 +155,15 @@ module Vmpooler
         end
         pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias')
 
-        redis.pipelined do
-          redis.hset("vmpooler__active__#{pool}", vm, Time.now)
-          redis.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
+        redis.pipelined do |pipeline|
+          pipeline.hset("vmpooler__active__#{pool}", vm, Time.now)
+          pipeline.hset("vmpooler__vm__#{vm}", 'checkout', Time.now)
           if ondemandrequest_hash['token:token']
-            redis.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token'])
-            redis.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user'])
-            redis.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i)
+            pipeline.hset("vmpooler__vm__#{vm}", 'token:token', ondemandrequest_hash['token:token'])
+            pipeline.hset("vmpooler__vm__#{vm}", 'token:user', ondemandrequest_hash['token:user'])
+            pipeline.hset("vmpooler__vm__#{vm}", 'lifetime', $config[:config]['vm_lifetime_auth'].to_i)
           end
-          redis.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
+          pipeline.sadd("vmpooler__#{request_id}__#{pool_alias}__#{pool}", vm)
         end
         move_vm_queue(pool, vm, 'pending', 'running', redis)
         check_ondemand_request_ready(request_id, redis)
@@ -164,12 +171,12 @@ module Vmpooler
         redis.smove("vmpooler__pending__#{pool}", "vmpooler__ready__#{pool}", vm)
       end
 
-      redis.pipelined do
-        redis.hset("vmpooler__boot__#{Date.today}", "#{pool}:#{vm}", finish) # maybe remove as this is never used by vmpooler itself?
-        redis.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
+      redis.pipelined do |pipeline|
+        pipeline.hset("vmpooler__boot__#{Date.today}", "#{pool}:#{vm}", finish) # maybe remove as this is never used by vmpooler itself?
+        pipeline.hset("vmpooler__vm__#{vm}", 'ready', Time.now)
 
         # last boot time is displayed in API, and used by alarming script
-        redis.hset('vmpooler__lastboot', pool, Time.now)
+        pipeline.hset('vmpooler__lastboot', pool, Time.now)
       end
 
       $metrics.timing("time_to_ready_state.#{pool}", finish)
@@ -361,35 +368,47 @@ module Vmpooler
       max_hostname_retries = 3
       while hostname_retries < max_hostname_retries
         hostname, hostname_available = generate_and_check_hostname
-        domain = $config[:config]['domain']
-        dns_ip, dns_available = check_dns_available(hostname, domain)
+        domain = Parsing.get_domain_for_pool(config, pool_name)
+        if domain
+          fqdn = "#{hostname}.#{domain}"
+        else
+          fqdn = hostname
+        end
+
+        # skip dns check if the provider is set to skip_dns_check_before_creating_vm
+        provider = get_provider_for_pool(pool_name)
+        if provider && provider.provider_config['skip_dns_check_before_creating_vm']
+          dns_available = true
+        else
+          dns_ip, dns_available = check_dns_available(fqdn)
+        end
+
         break if hostname_available && dns_available
 
         hostname_retries += 1
 
         if !hostname_available
           $metrics.increment("errors.duplicatehostname.#{pool_name}")
-          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
+          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} was not unique (attempt \##{hostname_retries} of #{max_hostname_retries})")
         elsif !dns_available
           $metrics.increment("errors.staledns.#{pool_name}")
-          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{hostname} already exists in DNS records (#{dns_ip}), stale DNS")
+          $logger.log('s', "[!] [#{pool_name}] Generated hostname #{fqdn} already exists in DNS records (#{dns_ip}), stale DNS")
         end
       end
 
-      raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{hostname}" unless hostname_available && dns_available
+      raise "Unable to generate a unique hostname after #{hostname_retries} attempts. The last hostname checked was #{fqdn}" unless hostname_available && dns_available
 
       hostname
     end
 
-    def check_dns_available(vm_name, domain = nil)
-      # Query the DNS for the name we want to create and if it already exists, mark it unavailable
-      # This protects against stale DNS records
-      vm_name = "#{vm_name}.#{domain}" if domain
+    # Query the DNS for the name we want to create and if it already exists, mark it unavailable
+    # This protects against stale DNS records
+    def check_dns_available(vm_name)
       begin
         dns_ip = Resolv.getaddress(vm_name)
       rescue Resolv::ResolvError
         # this is the expected case, swallow the error
-        # eg "no address for blah-daisy"
+        # eg "no address for blah-daisy.example.com"
         return ['', true]
       end
       [dns_ip, false]
@@ -397,6 +416,7 @@ module Vmpooler
 
     def _clone_vm(pool_name, provider, request_id = nil, pool_alias = nil)
       new_vmname = find_unique_hostname(pool_name)
+      pool_domain = Parsing.get_domain_for_pool(config, pool_name)
       mutex = vm_mutex(new_vmname)
       mutex.synchronize do
         @redis.with_metrics do |redis|
@@ -406,6 +426,7 @@ module Vmpooler
           redis.hset("vmpooler__vm__#{new_vmname}", 'clone', Time.now)
           redis.hset("vmpooler__vm__#{new_vmname}", 'template', pool_name) # This value is used to represent the pool.
           redis.hset("vmpooler__vm__#{new_vmname}", 'pool', pool_name)
+          redis.hset("vmpooler__vm__#{new_vmname}", 'domain', pool_domain) if pool_domain
           redis.hset("vmpooler__vm__#{new_vmname}", 'request_id', request_id) if request_id
           redis.hset("vmpooler__vm__#{new_vmname}", 'pool_alias', pool_alias) if pool_alias
           redis.exec
@@ -418,9 +439,9 @@ module Vmpooler
           finish = format('%<time>.2f', time: Time.now - start)
 
           @redis.with_metrics do |redis|
-            redis.pipelined do
-              redis.hset("vmpooler__clone__#{Date.today}", "#{pool_name}:#{new_vmname}", finish)
-              redis.hset("vmpooler__vm__#{new_vmname}", 'clone_time', finish)
+            redis.pipelined do |pipeline|
+              pipeline.hset("vmpooler__clone__#{Date.today}", "#{pool_name}:#{new_vmname}", finish)
+              pipeline.hset("vmpooler__vm__#{new_vmname}", 'clone_time', finish)
             end
           end
           $logger.log('s', "[+] [#{pool_name}] '#{new_vmname}' cloned in #{finish} seconds")
@@ -428,10 +449,10 @@ module Vmpooler
           $metrics.timing("clone.#{pool_name}", finish)
         rescue StandardError
           @redis.with_metrics do |redis|
-            redis.pipelined do
-              redis.srem("vmpooler__pending__#{pool_name}", new_vmname)
+            redis.pipelined do |pipeline|
+              pipeline.srem("vmpooler__pending__#{pool_name}", new_vmname)
               expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
-              redis.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
+              pipeline.expire("vmpooler__vm__#{new_vmname}", expiration_ttl)
             end
           end
           raise
@@ -462,12 +483,12 @@ module Vmpooler
 
       mutex.synchronize do
         @redis.with_metrics do |redis|
-          redis.pipelined do
-            redis.hdel("vmpooler__active__#{pool}", vm)
-            redis.hset("vmpooler__vm__#{vm}", 'destroy', Time.now)
+          redis.pipelined do |pipeline|
+            pipeline.hdel("vmpooler__active__#{pool}", vm)
+            pipeline.hset("vmpooler__vm__#{vm}", 'destroy', Time.now)
 
             # Auto-expire metadata key
-            redis.expire("vmpooler__vm__#{vm}", ($config[:redis]['data_ttl'].to_i * 60 * 60))
+            pipeline.expire("vmpooler__vm__#{vm}", ($config[:redis]['data_ttl'].to_i * 60 * 60))
           end
 
           start = Time.now
@@ -879,7 +900,7 @@ module Vmpooler
           loop_count = 1
           loop_delay = loop_delay_min
           provider = get_provider_for_pool(pool['name'])
-          raise("Could not find provider '#{pool['provider']}") if provider.nil?
+          raise("Could not find provider '#{pool['provider']}'") if provider.nil?
 
           sync_pool_template(pool)
           loop do
@@ -1204,19 +1225,19 @@ module Vmpooler
               pool_check_response[:destroyed_vms] += 1
               destroy_vm(vm, pool_name, provider)
             rescue StandardError => e
-              redis.pipelined do
-                redis.srem("vmpooler__completed__#{pool_name}", vm)
-                redis.hdel("vmpooler__active__#{pool_name}", vm)
-                redis.del("vmpooler__vm__#{vm}")
+              redis.pipelined do |pipeline|
+                pipeline.srem("vmpooler__completed__#{pool_name}", vm)
+                pipeline.hdel("vmpooler__active__#{pool_name}", vm)
+                pipeline.del("vmpooler__vm__#{vm}")
               end
               $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{e}")
             end
           else
             $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
-            redis.pipelined do
-              redis.srem("vmpooler__completed__#{pool_name}", vm)
-              redis.hdel("vmpooler__active__#{pool_name}", vm)
-              redis.del("vmpooler__vm__#{vm}")
+            redis.pipelined do |pipeline|
+              pipeline.srem("vmpooler__completed__#{pool_name}", vm)
+              pipeline.hdel("vmpooler__active__#{pool_name}", vm)
+              pipeline.del("vmpooler__vm__#{vm}")
             end
           end
         end
@@ -1366,7 +1387,7 @@ module Vmpooler
 
         return provider_klass.const_get(classname).new(config, logger, metrics, redis_connection_pool, provider_name, options)
       end
-      raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider.nil?
+      raise("Provider '#{provider_class}' is unknown for pool with provider name '#{provider_name}'") if provider_klass.nil?
     end
 
     def check_ondemand_requests(maxloop = 0,
@@ -1432,12 +1453,12 @@ module Vmpooler
       score = redis.zscore('vmpooler__provisioning__request', request_id)
       requested = requested.split(',')
 
-      redis.pipelined do
+      redis.pipelined do |pipeline|
         requested.each do |request|
-          redis.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
+          pipeline.zadd('vmpooler__odcreate__task', Time.now.to_i, "#{request}:#{request_id}")
         end
-        redis.zrem('vmpooler__provisioning__request', request_id)
-        redis.zadd('vmpooler__provisioning__processing', score, request_id)
+        pipeline.zrem('vmpooler__provisioning__request', request_id)
+        pipeline.zadd('vmpooler__provisioning__processing', score, request_id)
       end
     end
 
@@ -1467,9 +1488,9 @@ module Vmpooler
             redis.incr('vmpooler__tasks__ondemandclone')
             clone_vm(pool, provider, request_id, pool_alias)
           end
-          redis.pipelined do
-            redis.zrem(queue_key, request)
-            redis.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
+          redis.pipelined do |pipeline|
+            pipeline.zrem(queue_key, request)
+            pipeline.zadd(queue_key, score, "#{pool_alias}:#{pool}:#{remaining_count}:#{request_id}")
           end
         end
       end
@@ -1520,10 +1541,10 @@ module Vmpooler
 
       $logger.log('s', "Ondemand request for '#{request_id}' failed to provision all instances within the configured ttl '#{ondemand_request_ttl}'")
       expiration_ttl = $config[:redis]['data_ttl'].to_i * 60 * 60
-      redis.pipelined do
-        redis.zrem('vmpooler__provisioning__processing', request_id)
-        redis.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
-        redis.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
+      redis.pipelined do |pipeline|
+        pipeline.zrem('vmpooler__provisioning__processing', request_id)
+        pipeline.hset("vmpooler__odrequest__#{request_id}", 'status', 'failed')
+        pipeline.expire("vmpooler__odrequest__#{request_id}", expiration_ttl)
       end
       remove_vms_for_failed_request(request_id, expiration_ttl, redis)
       true
@@ -1533,11 +1554,11 @@ module Vmpooler
       request_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
       Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count|
         pools_filled = redis.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")
-        redis.pipelined do
+        redis.pipelined do |pipeline|
           pools_filled&.each do |vm|
-            move_vm_queue(pool, vm, 'running', 'completed', redis, "moved to completed queue. '#{request_id}' could not be filled in time")
+            move_vm_queue(pool, vm, 'running', 'completed', pipeline, "moved to completed queue. '#{request_id}' could not be filled in time")
           end
-          redis.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
+          pipeline.expire("vmpooler__#{request_id}__#{platform_alias}__#{pool}", expiration_ttl)
         end
       end
     end
@@ -1581,19 +1602,19 @@ module Vmpooler
       $config[:pools].each do |pool|
         provider_name = pool['provider']
         # The provider_class parameter can be defined in the provider's data eg
-        #:providers:
-        # :vsphere:
-        #  provider_class: 'vsphere'
-        # :another-vsphere:
-        #  provider_class: 'vsphere'
+        # :providers:
+        #   :vsphere:
+        #     provider_class: 'vsphere'
+        #   :another-vsphere:
+        #     provider_class: 'vsphere'
         # the above would create two providers/vsphere.rb class objects named 'vsphere' and 'another-vsphere'
         # each pools would then define which provider definition to use: vsphere or another-vsphere
         #
         # if provider_class is not defined it will try to use the provider_name as the class, this is to be
         # backwards compatible for example when there is only one provider listed
         # :providers:
-        #  :dummy:
-        #   filename: 'db.txs'
+        #   :dummy:
+        #     filename: 'db.txs'
         # the above example would create an object based on the class providers/dummy.rb
         if $config[:providers].nil? || $config[:providers][provider_name.to_sym].nil? || $config[:providers][provider_name.to_sym]['provider_class'].nil?
           provider_class = provider_name

data/lib/vmpooler/providers/base.rb

@@ -242,7 +242,7 @@ module Vmpooler
         # returns
         #   nil when successful. Raises error when encountered
         def create_template_delta_disks(_pool)
-          raise("#{self.class.name} does not implement create_template_delta_disks")
+          puts("#{self.class.name} does not implement create_template_delta_disks")
         end
 
         # inputs

data/lib/vmpooler/util/parsing.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# utility class shared between apps
+# utility class shared between apps api and pool_manager
 module Vmpooler
   class Parsing
     def self.get_platform_pool_count(requested, &_block)
@@ -12,5 +12,25 @@ module Vmpooler
         yield platform_alias, pool, count
       end
     end
+
+    # @param config [String] - the full config structure
+    # @param pool_name [String] - the name of the pool
+    # @return [String] - domain name for pool, if set in the provider for the pool or in the config block
+    def self.get_domain_for_pool(config, pool_name)
+      pool = config[:pools].find { |p| p['name'] == pool_name }
+      return nil unless pool
+
+      provider_name = pool.fetch('provider', 'vsphere') # see vmpooler.yaml.example where it states defaulting to vsphere
+
+      if config[:providers] && config[:providers][provider_name.to_sym] && config[:providers][provider_name.to_sym]['domain']
+        domain = config[:providers][provider_name.to_sym]['domain']
+      elsif config[:config] && config[:config]['domain']
+        domain = config[:config]['domain']
+      else
+        domain = nil
+      end
+
+      domain
+    end
   end
 end

data/lib/vmpooler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vmpooler
-  VERSION = '2.1.0'
+  VERSION = '2.4.0'
 end

data/lib/vmpooler.rb

@@ -3,6 +3,7 @@
 module Vmpooler
   require 'concurrent'
   require 'date'
+  require 'deep_merge'
   require 'json'
   require 'net/ldap'
   require 'open-uri'
@@ -16,6 +17,7 @@ module Vmpooler
 
   # Dependencies for tracing
   require 'opentelemetry-instrumentation-concurrent_ruby'
+  require 'opentelemetry-instrumentation-http_client'
   require 'opentelemetry-instrumentation-redis'
   require 'opentelemetry-instrumentation-sinatra'
   require 'opentelemetry-sdk'
@@ -41,8 +43,9 @@ module Vmpooler
       if parsed_config[:config]['extra_config']
         extra_configs = parsed_config[:config]['extra_config'].split(',')
         extra_configs.each do |config|
+          puts "loading extra_config file #{config}"
           extra_config = YAML.load_file(config)
-          parsed_config.merge!(extra_config)
+          parsed_config.deep_merge(extra_config)
         end
       end
     end
@@ -131,6 +134,7 @@ module Vmpooler
     # Create an index of pool aliases
     parsed_config[:pool_names] = Set.new
     unless parsed_config[:pools]
+      puts 'loading pools configuration from redis, since the config[:pools] is empty'
       redis = new_redis(parsed_config[:redis]['server'], parsed_config[:redis]['port'], parsed_config[:redis]['password'])
       parsed_config[:pools] = load_pools_from_redis(redis)
     end
@@ -264,6 +268,7 @@ module Vmpooler
     OpenTelemetry::SDK.configure do |c|
       c.use 'OpenTelemetry::Instrumentation::Sinatra'
       c.use 'OpenTelemetry::Instrumentation::ConcurrentRuby'
+      c.use 'OpenTelemetry::Instrumentation::HttpClient'
       c.use 'OpenTelemetry::Instrumentation::Redis'
 
       c.add_span_processor(span_processor)