vmpooler 3.3.0 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b586b8ee56526b89f47621d090f932159c46df74c6935002e8a71c18a02503fb
-  data.tar.gz: 51e4f7cf4dd4a5daa3130ab1bb60b08b4347196e1ba1c39f94a820d7356f2937
+  metadata.gz: 421d86151ba2c7a9bcc2310e3596b43f27a0280df1b7cb0c042dd12a63c80086
+  data.tar.gz: 7f8e2dea7fc3f4089995fc65022e98e48771f5a40cbbd30b51ddf162fad7b3f7
 SHA512:
-  metadata.gz: 39ba39f114252e74a08a6c3dbebd773cb2dda74adfc33216b7774dbf8fb83109f2a61076a3435cab54f8dc39adcdef60ee1cac44f24b3f3f441c7ceaf389e813
-  data.tar.gz: e295be65e92e61fad459d32d1a6171396b0599795355b99821d7bc8272ecbe19cfee78dd735c88e49653d5156afb7e529f12935d484fc2b1c479e5eed4be69e1
+  metadata.gz: 4722f36453f515e66cf69d2c9c19cd472dd289e012514b2823e6ccf2febbdc525d78969e4a24777779b2fd9dbf5c0f8137421e6fe0bdf95f726effead80cd114
+  data.tar.gz: 979d40b555d547d6ebec64efeb57af5cdb68eb90fbc6f759cf6a1bbfef074e66404b9e238ad410bd938cef611ae006e430aad86c754a50e3901d9c659081c733

data/lib/vmpooler/api/helpers.rb

@@ -68,7 +68,7 @@ module Vmpooler
         end
       end
 
-      def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str)
+      def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str, service_account_hash = nil)
         tracer.in_span(
           "Vmpooler::API::Helpers.#{__method__}",
           attributes: {
@@ -79,6 +79,14 @@ module Vmpooler
           },
           kind: :client
         ) do
+          if service_account_hash
+            username = service_account_hash[:user_dn]
+            password = service_account_hash[:password]
+          else
+            username = "#{user_object}=#{username_str},#{base}"
+            password = password_str
+          end
+
           ldap = Net::LDAP.new(
             :host => host,
             :port => port,
@@ -86,12 +94,22 @@ module Vmpooler
             :base => base,
             :auth => {
               :method => :simple,
-              :username => "#{user_object}=#{username_str},#{base}",
-              :password => password_str
+              :username => username,
+              :password => password
             }
           )
 
-          return true if ldap.bind
+          if service_account_hash
+            return true if ldap.bind_as(
+              :base => base,
+              :filter => "(#{user_object}=#{username_str})",
+              :password => password_str
+            )
+          elsif ldap.bind
+            return true
+          else
+            return false
+          end
 
           return false
         end
@@ -116,6 +134,7 @@ module Vmpooler
               :method => :start_tls,
               :tls_options => { :ssl_version => 'TLSv1' }
             }
+            service_account_hash = auth[:ldap]['service_account_hash']
 
             unless ldap_base.is_a? Array
               ldap_base = ldap_base.split
@@ -134,7 +153,8 @@ module Vmpooler
                   search_user_obj,
                   search_base,
                   username_str,
-                  password_str
+                  password_str,
+                  service_account_hash
                 )
                 return true if result
               end
@@ -551,18 +571,6 @@ module Vmpooler
           end
         end
       end
-
-      def vm_ready?(vm_name, domain = nil)
-        tracer.in_span("Vmpooler::API::Helpers.#{__method__}") do
-          begin
-            open_socket(vm_name, domain)
-          rescue StandardError => _e
-            return false
-          end
-
-          true
-        end
-      end
     end
   end
 end

data/lib/vmpooler/pool_manager.rb

@@ -82,31 +82,31 @@ module Vmpooler
     end
 
     # Check the state of a VM
-    def check_pending_vm(vm, pool, timeout, provider)
+    def check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       Thread.new do
         begin
-          _check_pending_vm(vm, pool, timeout, provider)
+          _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
         rescue StandardError => e
           $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
           @redis.with_metrics do |redis|
-            fail_pending_vm(vm, pool, timeout, redis)
+            fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
           end
           raise
         end
       end
     end
 
-    def _check_pending_vm(vm, pool, timeout, provider)
+    def _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       mutex = vm_mutex(vm)
       return if mutex.locked?
 
       mutex.synchronize do
         @redis.with_metrics do |redis|
           request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
-          if provider.vm_ready?(pool, vm)
+          if provider.vm_ready?(pool, vm, redis)
             move_pending_vm_to_ready(vm, pool, redis, request_id)
           else
-            fail_pending_vm(vm, pool, timeout, redis)
+            fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
           end
         end
       end
@@ -122,34 +122,53 @@ module Vmpooler
       $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
     end
 
-    def fail_pending_vm(vm, pool, timeout, redis, exists: true)
+    def fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
       clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
-
       time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
-      if time_since_clone > timeout
-        if exists
-          request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
-          pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
-          redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
-          if request_id
-            ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
-            if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
-              # will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
-              redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
-            end
-          end
-          $metrics.increment("errors.markedasfailed.#{pool}")
-          $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes")
-        else
+
+      already_timed_out = time_since_clone > timeout
+      timing_out_soon = time_since_clone > timeout_notification && !redis.hget("vmpooler__vm__#{vm}", 'timeout_notification')
+
+      if already_timed_out
+        unless exists
           remove_nonexistent_vm(vm, pool, redis)
+          return true
         end
+        open_socket_error = handle_timed_out_vm(vm, pool, redis)
       end
+
+      redis.hset("vmpooler__vm__#{vm}", 'timeout_notification', 1) if timing_out_soon
+
+      nonexist_warning = if already_timed_out
+                           "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: #{open_socket_error}"
+                         elsif timing_out_soon
+                           "[!] [#{pool}] '#{vm}' no longer exists when attempting to send notification of impending failure"
+                         else
+                           "[!] [#{pool}] '#{vm}' This error is wholly unexpected"
+                         end
+      $logger.log('d', nonexist_warning)
       true
     rescue StandardError => e
       $logger.log('d', "Fail pending VM failed with an error: #{e}")
       false
     end
 
+    def handle_timed_out_vm(vm, pool, redis)
+      request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
+      pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
+      open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
+      redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
+      if request_id
+        ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+        if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
+          # will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
+          redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
+        end
+      end
+      $metrics.increment("errors.markedasfailed.#{pool}")
+      open_socket_error
+    end
+
     def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
       clone_time = redis.hget("vmpooler__vm__#{vm}", 'clone')
       finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
@@ -197,11 +216,12 @@ module Vmpooler
 
     def vm_still_ready?(pool_name, vm_name, provider, redis)
       # Check if the VM is still ready/available
-      return true if provider.vm_ready?(pool_name, vm_name)
+      return true if provider.vm_ready?(pool_name, vm_name, redis)
 
       raise("VM #{vm_name} is not ready")
     rescue StandardError
-      move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "is unreachable, removed from 'ready' queue")
+      open_socket_error = redis.hget("vmpooler__vm__#{vm_name}", 'open_socket_error')
+      move_vm_queue(pool_name, vm_name, 'ready', 'completed', redis, "removed from 'ready' queue. vm unreachable with error: #{open_socket_error}")
     end
 
     def check_ready_vm(vm, pool_name, ttl, provider)
@@ -318,7 +338,7 @@ module Vmpooler
               redis.hset("vmpooler__vm__#{vm}", 'user_tagged', 'true') if success
             end
 
-            throw :stop_checking if provider.vm_ready?(pool, vm)
+            throw :stop_checking if provider.vm_ready?(pool, vm, redis)
 
             throw :stop_checking if provider.get_vm(pool, vm)
 
@@ -450,6 +470,9 @@ module Vmpooler
           ip_start = Time.now
           ip = provider.get_vm_ip_address(new_vmname, pool_name)
           ip_finish = format('%<time>.2f', time: Time.now - ip_start)
+
+          raise StandardError, "failed to obtain IP after #{ip_finish} seconds" if ip.nil?
+
           $logger.log('s', "[+] [#{pool_name}] Obtained IP for '#{new_vmname}' in #{ip_finish} seconds")
 
           @redis.with_metrics do |redis|
@@ -1248,19 +1271,20 @@ module Vmpooler
       end
     end
 
-    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
+    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout, pool_timeout_notification)
       pool_timeout ||= $config[:config]['timeout'] || 15
+      pool_timeout_notification ||= $config[:config]['timeout_notification'] || 5
       @redis.with_metrics do |redis|
         redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
           if inventory[vm]
             begin
               pool_check_response[:checked_pending_vms] += 1
-              check_pending_vm(vm, pool_name, pool_timeout, provider)
+              check_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, provider)
             rescue StandardError => e
               $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
             end
           else
-            fail_pending_vm(vm, pool_name, pool_timeout, redis, exists: false)
+            fail_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, redis, exists: false)
           end
         end
       end
@@ -1387,7 +1411,7 @@ module Vmpooler
 
       check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
 
-      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
+      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'], pool['timeout_notification'])
 
       check_completed_pool_vms(pool['name'], provider, pool_check_response, inventory)
 

data/lib/vmpooler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vmpooler
-  VERSION = '3.3.0'
+  VERSION = '3.5.0'
 end

data/lib/vmpooler.rb

@@ -82,6 +82,7 @@ module Vmpooler
     end
     parsed_config[:config]['clone_target']                       = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
     parsed_config[:config]['timeout']                            = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
+    parsed_config[:config]['timeout_notification']               = string_to_int(ENV['TIMEOUT_NOTIFICATION']) if ENV['TIMEOUT_NOTIFICATION']
     parsed_config[:config]['vm_lifetime_auth']                   = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
     parsed_config[:config]['max_tries']                          = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
     parsed_config[:config]['retry_factor']                       = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vmpooler
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.5.0
 platform: ruby
 authors:
 - Puppet
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-16 00:00:00.000000000 Z
+date: 2023-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement