vmpooler 3.4.0 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 928e0c76962d0ebbc7c3944ab16a0c03597f26e49b8132f4243110eae3794102
-  data.tar.gz: 89d63548f801b69578ad0c05333839e705efc848f97594af5e24e05ecff49f99
+  metadata.gz: abc6c0e799944bad6d5f766f21a5fd55038daf2741604a79dbf1dd409a4c6f54
+  data.tar.gz: 6977e9af54319329a9a526e0dfea5d39b453c57f391a24782f64fbf70f3cc300
 SHA512:
-  metadata.gz: ffc00f6b5d6f034aff3be29d48e14087a7e19202bf25c14a434f428848d9386dcca4a12a3add327b33843c36761adefe462ffa5f3017328eaca7bf4adfa2f6ab
-  data.tar.gz: 3d30c947ec9a3ce254435b4131bc7bf949740bf61aa30f297f35410a0297f3740e2fb8d5c2deb88bfaf4e6a662823afb66f51fa44e8547f54d76c72af34e48e7
+  metadata.gz: 33c3cea211a61f072498adbeff5c2ea8b9862e9dee8cac3a38b0f2757492a8df7ed7b8150b7213777b597804c235ea61c12eaeeed5a3c0907053d2f5111746a6
+  data.tar.gz: 28785ede40fdc245732195a6beeea9e4bced0c0d85fc54d12bfe6b6e931c50dc1d81942bd540201bcc33ff07ae876114b82524b47567314fbf1e1883e96252a6

data/lib/vmpooler/api/helpers.rb

@@ -68,7 +68,7 @@ module Vmpooler
         end
       end
 
-      def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str)
+      def authenticate_ldap(port, host, encryption_hash, user_object, base, username_str, password_str, service_account_hash = nil)
         tracer.in_span(
           "Vmpooler::API::Helpers.#{__method__}",
           attributes: {
@@ -79,6 +79,14 @@ module Vmpooler
           },
           kind: :client
         ) do
+          if service_account_hash
+            username = service_account_hash[:user_dn]
+            password = service_account_hash[:password]
+          else
+            username = "#{user_object}=#{username_str},#{base}"
+            password = password_str
+          end
+
           ldap = Net::LDAP.new(
             :host => host,
             :port => port,
@@ -86,12 +94,22 @@ module Vmpooler
             :base => base,
             :auth => {
               :method => :simple,
-              :username => "#{user_object}=#{username_str},#{base}",
-              :password => password_str
+              :username => username,
+              :password => password
             }
           )
 
-          return true if ldap.bind
+          if service_account_hash
+            return true if ldap.bind_as(
+              :base => base,
+              :filter => "(#{user_object}=#{username_str})",
+              :password => password_str
+            )
+          elsif ldap.bind
+            return true
+          else
+            return false
+          end
 
           return false
         end
@@ -116,6 +134,7 @@ module Vmpooler
               :method => :start_tls,
               :tls_options => { :ssl_version => 'TLSv1' }
             }
+            service_account_hash = auth[:ldap]['service_account_hash']
 
             unless ldap_base.is_a? Array
               ldap_base = ldap_base.split
@@ -134,7 +153,8 @@ module Vmpooler
                   search_user_obj,
                   search_base,
                   username_str,
-                  password_str
+                  password_str,
+                  service_account_hash
                 )
                 return true if result
               end

data/lib/vmpooler/api/v3.rb

@@ -283,11 +283,9 @@ module Vmpooler
       def update_user_metrics(operation, vmname)
         tracer.in_span("Vmpooler::API::V3.#{__method__}") do |span|
           begin
-            backend.multi
-            backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url')
-            backend.hget("vmpooler__vm__#{vmname}", 'token:user')
-            backend.hget("vmpooler__vm__#{vmname}", 'template')
-            jenkins_build_url, user, poolname = backend.exec
+            jenkins_build_url = backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url')
+            user = backend.hget("vmpooler__vm__#{vmname}", 'token:user')
+            poolname = backend.hget("vmpooler__vm__#{vmname}", 'template')
             poolname = poolname.gsub('.', '_')
 
             if user

data/lib/vmpooler/pool_manager.rb

@@ -82,21 +82,21 @@ module Vmpooler
     end
 
     # Check the state of a VM
-    def check_pending_vm(vm, pool, timeout, provider)
+    def check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       Thread.new do
         begin
-          _check_pending_vm(vm, pool, timeout, provider)
+          _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
         rescue StandardError => e
           $logger.log('s', "[!] [#{pool}] '#{vm}' #{timeout} #{provider} errored while checking a pending vm : #{e}")
           @redis.with_metrics do |redis|
-            fail_pending_vm(vm, pool, timeout, redis)
+            fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
           end
           raise
         end
       end
     end
 
-    def _check_pending_vm(vm, pool, timeout, provider)
+    def _check_pending_vm(vm, pool, timeout, timeout_notification, provider)
       mutex = vm_mutex(vm)
       return if mutex.locked?
 
@@ -106,7 +106,7 @@ module Vmpooler
           if provider.vm_ready?(pool, vm, redis)
             move_pending_vm_to_ready(vm, pool, redis, request_id)
           else
-            fail_pending_vm(vm, pool, timeout, redis)
+            fail_pending_vm(vm, pool, timeout, timeout_notification, redis)
           end
         end
       end
@@ -122,35 +122,55 @@ module Vmpooler
       $logger.log('d', "[!] [#{pool}] '#{vm}' no longer exists. Removing from pending.")
     end
 
-    def fail_pending_vm(vm, pool, timeout, redis, exists: true)
+    def fail_pending_vm(vm, pool, timeout, timeout_notification, redis, exists: true)
       clone_stamp = redis.hget("vmpooler__vm__#{vm}", 'clone')
-
       time_since_clone = (Time.now - Time.parse(clone_stamp)) / 60
-      if time_since_clone > timeout
-        if exists
-          request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
-          pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
-          open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
-          redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
-          if request_id
-            ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
-            if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
-              # will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
-              redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
-            end
-          end
-          $metrics.increment("errors.markedasfailed.#{pool}")
-          $logger.log('d', "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: #{open_socket_error}")
-        else
+
+      already_timed_out = time_since_clone > timeout
+      timing_out_soon = time_since_clone > timeout_notification && !redis.hget("vmpooler__vm__#{vm}", 'timeout_notification')
+
+      return true if !already_timed_out && !timing_out_soon
+
+      if already_timed_out
+        unless exists
           remove_nonexistent_vm(vm, pool, redis)
+          return true
         end
+        open_socket_error = handle_timed_out_vm(vm, pool, redis)
       end
+
+      redis.hset("vmpooler__vm__#{vm}", 'timeout_notification', 1) if timing_out_soon
+
+      nonexist_warning = if already_timed_out
+                           "[!] [#{pool}] '#{vm}' marked as 'failed' after #{timeout} minutes with error: #{open_socket_error}"
+                         elsif timing_out_soon
+                           "[!] [#{pool}] '#{vm}' no longer exists when attempting to send notification of impending failure"
+                         else
+                           "[!] [#{pool}] '#{vm}' This error is wholly unexpected"
+                         end
+      $logger.log('d', nonexist_warning)
       true
     rescue StandardError => e
       $logger.log('d', "Fail pending VM failed with an error: #{e}")
       false
     end
 
+    def handle_timed_out_vm(vm, pool, redis)
+      request_id = redis.hget("vmpooler__vm__#{vm}", 'request_id')
+      pool_alias = redis.hget("vmpooler__vm__#{vm}", 'pool_alias') if request_id
+      open_socket_error = redis.hget("vmpooler__vm__#{vm}", 'open_socket_error')
+      redis.smove("vmpooler__pending__#{pool}", "vmpooler__completed__#{pool}", vm)
+      if request_id
+        ondemandrequest_hash = redis.hgetall("vmpooler__odrequest__#{request_id}")
+        if ondemandrequest_hash && ondemandrequest_hash['status'] != 'failed' && ondemandrequest_hash['status'] != 'deleted'
+          # will retry a VM that did not come up as vm_ready? only if it has not been market failed or deleted
+          redis.zadd('vmpooler__odcreate__task', 1, "#{pool_alias}:#{pool}:1:#{request_id}")
+        end
+      end
+      $metrics.increment("errors.markedasfailed.#{pool}")
+      open_socket_error
+    end
+
     def move_pending_vm_to_ready(vm, pool, redis, request_id = nil)
       clone_time = redis.hget("vmpooler__vm__#{vm}", 'clone')
       finish = format('%<time>.2f', time: Time.now - Time.parse(clone_time))
@@ -452,6 +472,9 @@ module Vmpooler
           ip_start = Time.now
           ip = provider.get_vm_ip_address(new_vmname, pool_name)
           ip_finish = format('%<time>.2f', time: Time.now - ip_start)
+
+          raise StandardError, "failed to obtain IP after #{ip_finish} seconds" if ip.nil?
+
           $logger.log('s', "[+] [#{pool_name}] Obtained IP for '#{new_vmname}' in #{ip_finish} seconds")
 
           @redis.with_metrics do |redis|
@@ -1250,19 +1273,20 @@ module Vmpooler
       end
     end
 
-    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout)
+    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout, pool_timeout_notification)
       pool_timeout ||= $config[:config]['timeout'] || 15
+      pool_timeout_notification ||= $config[:config]['timeout_notification'] || 5
       @redis.with_metrics do |redis|
         redis.smembers("vmpooler__pending__#{pool_name}").reverse.each do |vm|
           if inventory[vm]
             begin
               pool_check_response[:checked_pending_vms] += 1
-              check_pending_vm(vm, pool_name, pool_timeout, provider)
+              check_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, provider)
             rescue StandardError => e
               $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{e}")
             end
           else
-            fail_pending_vm(vm, pool_name, pool_timeout, redis, exists: false)
+            fail_pending_vm(vm, pool_name, pool_timeout, pool_timeout_notification, redis, exists: false)
           end
         end
       end
@@ -1389,7 +1413,7 @@ module Vmpooler
 
       check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'] || $config[:config]['ready_ttl'])
 
-      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
+      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'], pool['timeout_notification'])
 
       check_completed_pool_vms(pool['name'], provider, pool_check_response, inventory)
 

data/lib/vmpooler/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vmpooler
-  VERSION = '3.4.0'
+  VERSION = '3.5.1'
 end

data/lib/vmpooler.rb

@@ -82,6 +82,7 @@ module Vmpooler
     end
     parsed_config[:config]['clone_target']                       = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
     parsed_config[:config]['timeout']                            = string_to_int(ENV['TIMEOUT']) if ENV['TIMEOUT']
+    parsed_config[:config]['timeout_notification']               = string_to_int(ENV['TIMEOUT_NOTIFICATION']) if ENV['TIMEOUT_NOTIFICATION']
     parsed_config[:config]['vm_lifetime_auth']                   = string_to_int(ENV['VM_LIFETIME_AUTH']) if ENV['VM_LIFETIME_AUTH']
     parsed_config[:config]['max_tries']                          = string_to_int(ENV['MAX_TRIES']) if ENV['MAX_TRIES']
     parsed_config[:config]['retry_factor']                       = string_to_int(ENV['RETRY_FACTOR']) if ENV['RETRY_FACTOR']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vmpooler
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.5.1
 platform: ruby
 authors:
 - Puppet
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-18 00:00:00.000000000 Z
+date: 2023-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement