vmpooler 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 033cdee6f8934932e9e717b8b229b36541fe2f88
-  data.tar.gz: d798c6417320e4942bf0965ed3c1e2d303e0bab6
+  metadata.gz: a1b32e5280b92e461243bc99e275e1637fe3c644
+  data.tar.gz: 702ee61515185102a99cecc1d1475b84f2c19ada
 SHA512:
-  metadata.gz: a45de99b68aaf08784fbfac2793a9da60d38c9a8af20ae802362015841e6962dfb2ab027dbfcc7792d8808130752991c3419e95a4ea0764dc625c8a1b97dd5d0
-  data.tar.gz: 68dbfedba323a249e0a0bace756f2aec21ce1e9a770482b477ea9b197f9ff7a86ba55cdc29a9493756f713c514689f1db66a5196ea01f8f42126315e726f1daa
+  metadata.gz: 9566067bbaee8024bdcd05d715082e158a61503de81e9d35a7600ba9567e4f033638350718143c86c17db0e912a1fb42f42032b465350afba666a7f3856c8acb
+  data.tar.gz: 415ebd0a8bbbd125ec0a0972be930197f88ac1d927f41683e526aec4465406fe0a6954750892b0b1991b16046686ad64420bb103821620e68e8a017a9b37f208

data/lib/vmpooler.rb

@@ -1,15 +1,16 @@
 module Vmpooler
   require 'date'
   require 'json'
-  require 'open-uri'
   require 'net/ldap'
+  require 'open-uri'
+  require 'pickup'
   require 'rbvmomi'
   require 'redis'
+  require 'set'
   require 'sinatra/base'
   require 'time'
   require 'timeout'
   require 'yaml'
-  require 'set'
 
   %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"

data/lib/vmpooler/api/v1.rb

@@ -37,15 +37,38 @@ module Vmpooler
     end
 
     def fetch_single_vm(template)
-      vm = backend.spop('vmpooler__ready__' + template)
-      return [vm, template] if vm
-
+      template_backends = [template]
       aliases = Vmpooler::API.settings.config[:alias]
-      if aliases && aliased_template = aliases[template]
-        vm = backend.spop('vmpooler__ready__' + aliased_template)
-        return [vm, aliased_template] if vm
+      if aliases
+        template_backends << aliases[template] if aliases[template]
+
+        pool_index = pool_index(pools)
+        weighted_pools = {}
+        template_backends.each do |t|
+          next unless pool_index.key? t
+          index = pool_index[t]
+          clone_target = pools[index]['clone_target'] || config['clone_target']
+          next unless config.key?('backend_weight')
+          weight = config['backend_weight'][clone_target]
+          if weight
+            weighted_pools[t] = weight
+          end
+        end
+
+        if weighted_pools.count == template_backends.count
+          pickup = Pickup.new(weighted_pools)
+          selection = pickup.pick
+          template_backends.delete(selection)
+          template_backends.unshift(selection)
+        else
+          template_backends = template_backends.sample(template_backends.count)
+        end
       end
 
+      template_backends.each do |t|
+        vm = backend.spop('vmpooler__ready__' + t)
+        return [vm, t] if vm
+      end
       [nil, nil]
     end
 

data/lib/vmpooler/pool_manager.rb

@@ -188,8 +188,10 @@ module Vmpooler
 
       # Check if the hostname has magically changed from underneath Pooler
       vm_hash = provider.get_vm(pool['name'], vm)
+      return unless vm_hash.is_a? Hash
       hostname = vm_hash['hostname']
 
+      return if hostname.nil?
       return if hostname.empty?
       return if hostname == vm
       $redis.smove('vmpooler__ready__' + pool['name'], 'vmpooler__completed__' + pool['name'], vm)
@@ -243,20 +245,18 @@ module Vmpooler
     end
 
     # Clone a VM
-    def clone_vm(pool, provider)
+    def clone_vm(pool_name, provider)
       Thread.new do
         begin
-          _clone_vm(pool, provider)
+          _clone_vm(pool_name, provider)
         rescue => err
-          $logger.log('s', "[!] [#{pool['name']}] failed while cloning VM with an error: #{err}")
+          $logger.log('s', "[!] [#{pool_name}] failed while cloning VM with an error: #{err}")
           raise
         end
       end
     end
 
-    def _clone_vm(pool, provider)
-      pool_name = pool['name']
-
+    def _clone_vm(pool_name, provider)
       # Generate a randomized hostname
       o = [('a'..'z'), ('0'..'9')].map(&:to_a).flatten
       new_vmname = $config[:config]['prefix'] + o[rand(25)] + (0...14).map { o[rand(o.length)] }.join
@@ -723,7 +723,16 @@ module Vmpooler
     end
 
     def prepare_template(pool, provider)
-      provider.create_template_delta_disks(pool) if $config[:config]['create_template_delta_disks']
+      if $config[:config]['create_template_delta_disks']
+        unless $redis.sismember('vmpooler__template__deltas', pool['template'])
+          begin
+            provider.create_template_delta_disks(pool)
+            $redis.sadd('vmpooler__template__deltas', pool['template'])
+          rescue => err
+            $logger.log('s', "[!] [#{pool['name']}] failed while preparing a template with an error. As a result vmpooler could not create the template delta disks. Either a template delta disk already exists, or the template delta disk creation failed. The error is: #{err}")
+          end
+        end
+      end
       $redis.hset('vmpooler__template__prepared', pool['name'], pool['template'])
     end
 
@@ -779,7 +788,9 @@ module Vmpooler
       $logger.log('s', "[*] [#{pool['name']}] is ready for use")
     end
 
-    def remove_excess_vms(pool, provider, ready, total)
+    def remove_excess_vms(pool)
+      ready = $redis.scard("vmpooler__ready__#{pool['name']}")
+      total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
       return if total.nil?
       return if total == 0
       mutex = pool_mutex(pool['name'])
@@ -811,17 +822,7 @@ module Vmpooler
       end
     end
 
-    def _check_pool(pool, provider)
-      pool_check_response = {
-        discovered_vms: 0,
-        checked_running_vms: 0,
-        checked_ready_vms: 0,
-        checked_pending_vms: 0,
-        destroyed_vms: 0,
-        migrated_vms: 0,
-        cloned_vms: 0
-      }
-      # INVENTORY
+    def create_inventory(pool, provider, pool_check_response)
       inventory = {}
       begin
         mutex = pool_mutex(pool['name'])
@@ -844,103 +845,170 @@ module Vmpooler
           end
         end
       rescue => err
-        $logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while inspecting inventory: #{err}")
-        return pool_check_response
+        $logger.log('s', "[!] [#{pool['name']}] _check_pool failed with an error while running create_inventory: #{err}")
+        raise(err)
       end
+      inventory
+    end
 
-      # RUNNING
-      $redis.smembers("vmpooler__running__#{pool['name']}").each do |vm|
+    def check_running_pool_vms(pool_name, provider, pool_check_response, inventory)
+      $redis.smembers("vmpooler__running__#{pool_name}").each do |vm|
         if inventory[vm]
           begin
             vm_lifetime = $redis.hget('vmpooler__vm__' + vm, 'lifetime') || $config[:config]['vm_lifetime'] || 12
             pool_check_response[:checked_running_vms] += 1
-            check_running_vm(vm, pool['name'], vm_lifetime, provider)
+            check_running_vm(vm, pool_name, vm_lifetime, provider)
           rescue => err
-            $logger.log('d', "[!] [#{pool['name']}] _check_pool with an error while evaluating running VMs: #{err}")
+            $logger.log('d', "[!] [#{pool_name}] _check_pool with an error while evaluating running VMs: #{err}")
           end
         else
-          move_vm_queue(pool['name'], vm, 'running', 'completed', 'is a running VM but is missing from inventory.  Marking as completed.')
+          move_vm_queue(pool_name, vm, 'running', 'completed', 'is a running VM but is missing from inventory.  Marking as completed.')
         end
       end
+    end
 
-      # READY
-      $redis.smembers("vmpooler__ready__#{pool['name']}").each do |vm|
+    def check_ready_pool_vms(pool_name, provider, pool_check_response, inventory, pool_ttl = 0)
+      $redis.smembers("vmpooler__ready__#{pool_name}").each do |vm|
         if inventory[vm]
           begin
             pool_check_response[:checked_ready_vms] += 1
-            check_ready_vm(vm, pool, pool['ready_ttl'] || 0, provider)
+            check_ready_vm(vm, pool_name, pool_ttl || 0, provider)
           rescue => err
-            $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating ready VMs: #{err}")
+            $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating ready VMs: #{err}")
           end
         else
-          move_vm_queue(pool['name'], vm, 'ready', 'completed', 'is a ready VM but is missing from inventory.  Marking as completed.')
+          move_vm_queue(pool_name, vm, 'ready', 'completed', 'is a ready VM but is missing from inventory.  Marking as completed.')
         end
       end
+    end
 
-      # PENDING
-      $redis.smembers("vmpooler__pending__#{pool['name']}").each do |vm|
-        pool_timeout = pool['timeout'] || $config[:config]['timeout'] || 15
+    def check_pending_pool_vms(pool_name, provider, pool_check_response, inventory, pool_timeout = nil)
+      pool_timeout ||= $config[:config]['timeout'] || 15
+      $redis.smembers("vmpooler__pending__#{pool_name}").each do |vm|
         if inventory[vm]
           begin
             pool_check_response[:checked_pending_vms] += 1
-            check_pending_vm(vm, pool['name'], pool_timeout, provider)
+            check_pending_vm(vm, pool_name, pool_timeout, provider)
           rescue => err
-            $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating pending VMs: #{err}")
+            $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating pending VMs: #{err}")
           end
         else
-          fail_pending_vm(vm, pool['name'], pool_timeout, false)
+          fail_pending_vm(vm, pool_name, pool_timeout, false)
         end
       end
+    end
 
-      # COMPLETED
-      $redis.smembers("vmpooler__completed__#{pool['name']}").each do |vm|
+    def check_completed_pool_vms(pool_name, provider, pool_check_response, inventory)
+      $redis.smembers("vmpooler__completed__#{pool_name}").each do |vm|
         if inventory[vm]
           begin
             pool_check_response[:destroyed_vms] += 1
-            destroy_vm(vm, pool['name'], provider)
+            destroy_vm(vm, pool_name, provider)
           rescue => err
-            $redis.srem("vmpooler__completed__#{pool['name']}", vm)
-            $redis.hdel("vmpooler__active__#{pool['name']}", vm)
+            $redis.srem("vmpooler__completed__#{pool_name}", vm)
+            $redis.hdel("vmpooler__active__#{pool_name}", vm)
             $redis.del("vmpooler__vm__#{vm}")
-            $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating completed VMs: #{err}")
+            $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating completed VMs: #{err}")
           end
         else
-          $logger.log('s', "[!] [#{pool['name']}] '#{vm}' not found in inventory, removed from 'completed' queue")
-          $redis.srem("vmpooler__completed__#{pool['name']}", vm)
-          $redis.hdel("vmpooler__active__#{pool['name']}", vm)
+          $logger.log('s', "[!] [#{pool_name}] '#{vm}' not found in inventory, removed from 'completed' queue")
+          $redis.srem("vmpooler__completed__#{pool_name}", vm)
+          $redis.hdel("vmpooler__active__#{pool_name}", vm)
           $redis.del("vmpooler__vm__#{vm}")
         end
       end
+    end
 
-      # DISCOVERED
+    def check_discovered_pool_vms(pool_name)
       begin
-        $redis.smembers("vmpooler__discovered__#{pool['name']}").each do |vm|
+        $redis.smembers("vmpooler__discovered__#{pool_name}").each do |vm|
           %w[pending ready running completed].each do |queue|
-            if $redis.sismember("vmpooler__#{queue}__#{pool['name']}", vm)
-              $logger.log('d', "[!] [#{pool['name']}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
-              $redis.srem("vmpooler__discovered__#{pool['name']}", vm)
+            if $redis.sismember("vmpooler__#{queue}__#{pool_name}", vm)
+              $logger.log('d', "[!] [#{pool_name}] '#{vm}' found in '#{queue}', removed from 'discovered' queue")
+              $redis.srem("vmpooler__discovered__#{pool_name}", vm)
             end
           end
 
-          if $redis.sismember("vmpooler__discovered__#{pool['name']}", vm)
-            $redis.smove("vmpooler__discovered__#{pool['name']}", "vmpooler__completed__#{pool['name']}", vm)
+          if $redis.sismember("vmpooler__discovered__#{pool_name}", vm)
+            $redis.smove("vmpooler__discovered__#{pool_name}", "vmpooler__completed__#{pool_name}", vm)
           end
         end
       rescue => err
-        $logger.log('d', "[!] [#{pool['name']}] _check_pool failed with an error while evaluating discovered VMs: #{err}")
+        $logger.log('d', "[!] [#{pool_name}] _check_pool failed with an error while evaluating discovered VMs: #{err}")
       end
+    end
 
-      # MIGRATIONS
-      $redis.smembers("vmpooler__migrating__#{pool['name']}").each do |vm|
+    def check_migrating_pool_vms(pool_name, provider, pool_check_response, inventory)
+      $redis.smembers("vmpooler__migrating__#{pool_name}").each do |vm|
         if inventory[vm]
           begin
             pool_check_response[:migrated_vms] += 1
-            migrate_vm(vm, pool['name'], provider)
+            migrate_vm(vm, pool_name, provider)
+          rescue => err
+            $logger.log('s', "[x] [#{pool_name}] '#{vm}' failed to migrate: #{err}")
+          end
+        end
+      end
+    end
+
+    def repopulate_pool_vms(pool_name, provider, pool_check_response, pool_size)
+      return if pool_mutex(pool_name).locked?
+      ready = $redis.scard("vmpooler__ready__#{pool_name}")
+      total = $redis.scard("vmpooler__pending__#{pool_name}") + ready
+
+      $metrics.gauge("ready.#{pool_name}", $redis.scard("vmpooler__ready__#{pool_name}"))
+      $metrics.gauge("running.#{pool_name}", $redis.scard("vmpooler__running__#{pool_name}"))
+
+      if $redis.get("vmpooler__empty__#{pool_name}")
+        $redis.del("vmpooler__empty__#{pool_name}") unless ready.zero?
+      elsif ready.zero?
+        $redis.set("vmpooler__empty__#{pool_name}", 'true')
+        $logger.log('s', "[!] [#{pool_name}] is empty")
+      end
+
+      (pool_size - total).times do
+        if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
+          begin
+            $redis.incr('vmpooler__tasks__clone')
+            pool_check_response[:cloned_vms] += 1
+            clone_vm(pool_name, provider)
           rescue => err
-            $logger.log('s', "[x] [#{pool['name']}] '#{vm}' failed to migrate: #{err}")
+            $logger.log('s', "[!] [#{pool_name}] clone failed during check_pool with an error: #{err}")
+            $redis.decr('vmpooler__tasks__clone')
+            raise
           end
         end
       end
+    end
+
+    def _check_pool(pool, provider)
+      pool_check_response = {
+        discovered_vms: 0,
+        checked_running_vms: 0,
+        checked_ready_vms: 0,
+        checked_pending_vms: 0,
+        destroyed_vms: 0,
+        migrated_vms: 0,
+        cloned_vms: 0
+      }
+
+      begin
+        inventory = create_inventory(pool, provider, pool_check_response)
+      rescue => err
+        return(pool_check_response)
+      end
+
+      check_running_pool_vms(pool['name'], provider, pool_check_response, inventory)
+
+      check_ready_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['ready_ttl'])
+
+      check_pending_pool_vms(pool['name'], provider, pool_check_response, inventory, pool['timeout'])
+
+      check_completed_pool_vms(pool['name'], provider, pool_check_response, inventory)
+
+      check_discovered_pool_vms(pool['name'])
+
+      check_migrating_pool_vms(pool['name'], provider, pool_check_response, inventory)
 
       # UPDATE TEMPLATE
       # Evaluates a pool template to ensure templates are prepared adequately for the configured provider
@@ -948,46 +1016,15 @@ module Vmpooler
       # Additionally, a pool will drain ready and pending instances
       evaluate_template(pool, provider)
 
-      # REPOPULATE
-      # Do not attempt to repopulate a pool while a template is updating
-      unless pool_mutex(pool['name']).locked?
-        ready = $redis.scard("vmpooler__ready__#{pool['name']}")
-        total = $redis.scard("vmpooler__pending__#{pool['name']}") + ready
+      # Check to see if a pool size change has been made via the configuration API
+      # Since check_pool runs in a loop it does not
+      # otherwise identify this change when running
+      update_pool_size(pool)
 
-        $metrics.gauge("ready.#{pool['name']}", $redis.scard("vmpooler__ready__#{pool['name']}"))
-        $metrics.gauge("running.#{pool['name']}", $redis.scard("vmpooler__running__#{pool['name']}"))
-
-        if $redis.get("vmpooler__empty__#{pool['name']}")
-          $redis.del("vmpooler__empty__#{pool['name']}") unless ready.zero?
-        elsif ready.zero?
-          $redis.set("vmpooler__empty__#{pool['name']}", 'true')
-          $logger.log('s', "[!] [#{pool['name']}] is empty")
-        end
-
-        # Check to see if a pool size change has been made via the configuration API
-        # Since check_pool runs in a loop it does not
-        # otherwise identify this change when running
-        update_pool_size(pool)
-
-        if total < pool['size']
-          (1..(pool['size'] - total)).each do |_i|
-            if $redis.get('vmpooler__tasks__clone').to_i < $config[:config]['task_limit'].to_i
-              begin
-                $redis.incr('vmpooler__tasks__clone')
-                pool_check_response[:cloned_vms] += 1
-                clone_vm(pool, provider)
-              rescue => err
-                $logger.log('s', "[!] [#{pool['name']}] clone failed during check_pool with an error: #{err}")
-                $redis.decr('vmpooler__tasks__clone')
-                raise
-              end
-            end
-          end
-        end
-      end
+      repopulate_pool_vms(pool['name'], provider, pool_check_response, pool['size'])
 
       # Remove VMs in excess of the configured pool size
-      remove_excess_vms(pool, provider, ready, total)
+      remove_excess_vms(pool)
 
       pool_check_response
     end

data/lib/vmpooler/providers/vsphere.rb

@@ -428,7 +428,7 @@ module Vmpooler
 
         def vm_ready?(_pool_name, vm_name)
           begin
-            open_socket(vm_name)
+            open_socket(vm_name, global_config[:config]['domain'])
           rescue => _err
             return false
           end

data/lib/vmpooler/version.rb

@@ -1,3 +1,3 @@
 module Vmpooler
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.2.1'.freeze
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: vmpooler
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-24 00:00:00.000000000 Z
+date: 2018-09-19 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: pickup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.11
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement