vmpooler 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c0edeb60c2a23b1dc15da9f2535b9420b4de8c3
-  data.tar.gz: 667639acb53c6b83bb530a0134b506da57e51a32
+  metadata.gz: 033cdee6f8934932e9e717b8b229b36541fe2f88
+  data.tar.gz: d798c6417320e4942bf0965ed3c1e2d303e0bab6
 SHA512:
-  metadata.gz: 647a7292fb7b5c9bc991ce68e2d42998a6604bd77392221baca916c90ca696e58cd6879a4125ce863c97b5ff04dfca98ed6d5d7380c7d9251ff663c8eca3d61b
-  data.tar.gz: bf66abbb2ea1b1ba8e06e82ecaa293f2e160fab25959ce260f1b066b8ddeb5185c3c1fa40b14d3fc39f6f009bdf53b8dbaa8a64ff3380249ddea31af1c30a035
+  metadata.gz: a45de99b68aaf08784fbfac2793a9da60d38c9a8af20ae802362015841e6962dfb2ab027dbfcc7792d8808130752991c3419e95a4ea0764dc625c8a1b97dd5d0
+  data.tar.gz: 68dbfedba323a249e0a0bace756f2aec21ce1e9a770482b477ea9b197f9ff7a86ba55cdc29a9493756f713c514689f1db66a5196ea01f8f42126315e726f1daa

data/lib/vmpooler/pool_manager.rb

@@ -1,3 +1,5 @@
+require 'vmpooler/providers'
+
 module Vmpooler
   class PoolManager
     CHECK_LOOP_DELAY_MIN_DEFAULT = 5
@@ -26,6 +28,9 @@ module Vmpooler
       @reconfigure_pool = {}
 
       @vm_mutex = {}
+
+      # load specified providers from config file
+      load_used_providers
     end
 
     def config
@@ -298,7 +303,6 @@ module Vmpooler
       mutex = vm_mutex(vm)
       return if mutex.locked?
       mutex.synchronize do
-        $redis.srem('vmpooler__completed__' + pool, vm)
         $redis.hdel('vmpooler__active__' + pool, vm)
         $redis.hset('vmpooler__vm__' + vm, 'destroy', Time.now)
 
@@ -309,10 +313,60 @@ module Vmpooler
 
         provider.destroy_vm(pool, vm)
 
+        $redis.srem('vmpooler__completed__' + pool, vm)
+
         finish = format('%.2f', Time.now - start)
         $logger.log('s', "[-] [#{pool}] '#{vm}' destroyed in #{finish} seconds")
         $metrics.timing("destroy.#{pool}", finish)
       end
+      dereference_mutex(vm)
+    end
+
+    def purge_unused_vms_and_folders
+      global_purge = $config[:config]['purge_unconfigured_folders']
+      providers = $config[:providers].keys
+      providers.each do |provider|
+        provider_purge = $config[:providers][provider]['purge_unconfigured_folders']
+        provider_purge = global_purge if provider_purge.nil?
+        if provider_purge
+          Thread.new do
+            begin
+              purge_vms_and_folders($providers[provider.to_s])
+            rescue => err
+              $logger.log('s', "[!] failed while purging provider #{provider.to_s} VMs and folders with an error: #{err}")
+            end
+          end
+        end
+      end
+      return
+    end
+
+    # Return a list of pool folders
+    def pool_folders(provider)
+      provider_name = provider.name
+      folders = {}
+      $config[:pools].each do |pool|
+        next unless pool['provider'] == provider_name
+        folder_parts = pool['folder'].split('/')
+        datacenter = provider.get_target_datacenter_from_config(pool['name'])
+        folders[folder_parts.pop] = "#{datacenter}/vm/#{folder_parts.join('/')}"
+      end
+      folders
+    end
+
+    def get_base_folders(folders)
+      base = []
+      folders.each do |key, value|
+        base << value
+      end
+      base.uniq
+    end
+
+    def purge_vms_and_folders(provider)
+      configured_folders = pool_folders(provider)
+      base_folders = get_base_folders(configured_folders)
+      whitelist = provider.provider_config['folder_whitelist']
+      provider.purge_unconfigured_folders(base_folders, configured_folders, whitelist)
     end
 
     def create_vm_disk(pool_name, vm, disk_size, provider)
@@ -408,19 +462,36 @@ module Vmpooler
       result
     end
 
+    # load only providers used in config file
+    def load_used_providers
+      Vmpooler::Providers.load_by_name(used_providers)
+    end
+
+    # @return [Array] - a list of used providers from the config file, defaults to the default providers
+    # ie. ["vsphere", "dummy"]
+    def used_providers
+      pools = config[:pools] || []
+      @used_providers ||= (pools.map { |pool| pool[:provider] || pool['provider'] }.compact + default_providers ).uniq
+    end
+
+    # @return [Array] - returns a list of providers that should always be loaded
+    # note: vsphere is the default if user does not specify although this should not be
+    # if vsphere is to no longer be loaded by default please remove
+    def default_providers
+      @default_providers ||= %w( vsphere dummy )
+    end
+
     def get_pool_name_for_vm(vm_name)
       # the 'template' is a bad name.  Should really be 'poolname'
       $redis.hget('vmpooler__vm__' + vm_name, 'template')
     end
 
+    # @param pool_name [String] - the name of the pool
+    # @return [Provider] - returns the provider class Object
     def get_provider_for_pool(pool_name)
-      provider_name = nil
-      $config[:pools].each do |pool|
-        next unless pool['name'] == pool_name
-        provider_name = pool['provider']
-      end
-      return nil if provider_name.nil?
-
+      pool = $config[:pools].find { |pool| pool['name'] == pool_name }
+      return nil unless pool
+      provider_name = pool.fetch('provider', nil)
       $providers[provider_name]
     end
 
@@ -634,6 +705,14 @@ module Vmpooler
       @vm_mutex[vmname] || @vm_mutex[vmname] = Mutex.new
     end
 
+    def dereference_mutex(vmname)
+      if @vm_mutex.delete(vmname)
+        return true
+      else
+        return
+      end
+    end
+
     def sync_pool_template(pool)
       pool_template = $redis.hget('vmpooler__config__template', pool['name'])
       if pool_template
@@ -992,6 +1071,8 @@ module Vmpooler
         end
       end
 
+      purge_unused_vms_and_folders
+
       loop_count = 1
       loop do
         if !$threads['disk_manager']

data/lib/vmpooler/providers/base.rb

@@ -225,6 +225,18 @@ module Vmpooler
         def create_template_delta_disks(pool)
           raise("#{self.class.name} does not implement create_template_delta_disks")
         end
+
+        # inputs
+        #   [String] provider_name : Name of the provider
+        # returns
+        #   Hash of folders
+        def get_target_datacenter_from_config(provider_name)
+          raise("#{self.class.name} does not implement get_target_datacenter_from_config")
+        end
+
+        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+          raise("#{self.class.name} does not implement purge_unconfigured_folders")
+        end
       end
     end
   end

data/lib/vmpooler/providers/dummy.rb

@@ -1,4 +1,5 @@
 require 'yaml'
+require 'vmpooler/providers/base'
 
 module Vmpooler
   class PoolManager

data/lib/vmpooler/providers/vsphere.rb

@@ -1,3 +1,5 @@
+require 'vmpooler/providers/base'
+
 module Vmpooler
   class PoolManager
     class Provider
@@ -42,6 +44,109 @@ module Vmpooler
           'vsphere'
         end
 
+        def folder_configured?(folder_title, base_folder, configured_folders, whitelist)
+          if whitelist
+            return true if whitelist.include?(folder_title)
+          end
+          return false unless configured_folders.keys.include?(folder_title)
+          return false unless configured_folders[folder_title] == base_folder
+          return true
+        end
+
+        def destroy_vm_and_log(vm_name, vm_object, pool, data_ttl)
+          try = 0 if try.nil?
+          max_tries = 3
+          $redis.srem("vmpooler__completed__#{pool}", vm_name)
+          $redis.hdel("vmpooler__active__#{pool}", vm_name)
+          $redis.hset("vmpooler__vm__#{vm_name}", 'destroy', Time.now)
+
+          # Auto-expire metadata key
+          $redis.expire('vmpooler__vm__' + vm_name, (data_ttl * 60 * 60))
+
+          start = Time.now
+
+          if vm_object.is_a? RbVmomi::VIM::Folder
+            logger.log('s', "[!] [#{pool}] '#{vm_name}' is a folder, bailing on destroying")
+            raise('Expected VM, but received a folder object')
+          end
+          vm_object.PowerOffVM_Task.wait_for_completion if vm_object.runtime && vm_object.runtime.powerState && vm_object.runtime.powerState == 'poweredOn'
+          vm_object.Destroy_Task.wait_for_completion
+
+          finish = format('%.2f', Time.now - start)
+          logger.log('s', "[-] [#{pool}] '#{vm_name}' destroyed in #{finish} seconds")
+          metrics.timing("destroy.#{pool}", finish)
+        rescue RuntimeError
+          raise
+        rescue => err
+          try += 1
+          logger.log('s', "[!] [#{pool}] failed to destroy '#{vm_name}' with an error: #{err}")
+          try >= max_tries ? raise : retry
+        end
+
+        def destroy_folder_and_children(folder_object)
+          vms = {}
+          data_ttl = $config[:redis]['data_ttl'].to_i
+          folder_name = folder_object.name
+          unless folder_object.childEntity.count == 0
+            folder_object.childEntity.each do |vm|
+              vms[vm.name] = vm
+            end
+
+            vms.each do |vm_name, vm_object|
+              destroy_vm_and_log(vm_name, vm_object, folder_name, data_ttl)
+            end
+          end
+          destroy_folder(folder_object)
+        end
+
+        def destroy_folder(folder_object)
+          try = 0 if try.nil?
+          max_tries = 3
+          logger.log('s', "[-] [#{folder_object.name}] removing unconfigured folder")
+          folder_object.Destroy_Task.wait_for_completion
+        rescue
+          try += 1
+          try >= max_tries ? raise : retry
+        end
+
+        def purge_unconfigured_folders(base_folders, configured_folders, whitelist)
+          @connection_pool.with_metrics do |pool_object|
+            connection = ensured_vsphere_connection(pool_object)
+
+            base_folders.each do |base_folder|
+              folder_children = get_folder_children(base_folder, connection)
+              unless folder_children.empty?
+                folder_children.each do |folder_hash|
+                  folder_hash.each do |folder_title, folder_object|
+                    unless folder_configured?(folder_title, base_folder, configured_folders, whitelist)
+                      destroy_folder_and_children(folder_object)
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def get_folder_children(folder_name, connection)
+          folders = []
+
+          propSpecs = {
+            :entity => self,
+            :inventoryPath => folder_name
+          }
+          folder_object = connection.searchIndex.FindByInventoryPath(propSpecs)
+
+          return folders if folder_object.nil?
+
+          folder_object.childEntity.each do |folder|
+            next unless folder.is_a? RbVmomi::VIM::Folder
+            folders << { folder.name => folder }
+          end
+
+          folders
+        end
+
         def vms_in_pool(pool_name)
           vms = []
           @connection_pool.with_metrics do |pool_object|

data/lib/vmpooler/providers.rb

@@ -1,7 +1,119 @@
-%w[base dummy vsphere].each do |lib|
-  begin
-    require "vmpooler/providers/#{lib}"
-  rescue LoadError
-    require File.expand_path(File.join(File.dirname(__FILE__), 'providers', lib))
+require 'pathname'
+
+module Vmpooler
+  class Providers
+
+    # @param names [Array] - an array of names or string name of a provider
+    # @return [Array] - list of provider files loaded
+    # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"]
+    def self.load_by_name(names)
+      names = Array(names)
+      instance = self.new
+      names.map {|name| instance.load_from_gems(name)}.flatten
+    end
+
+    # @return [Array] - array of provider files
+    # ie. ["lib/vmpooler/providers/base.rb", "lib/vmpooler/providers/dummy.rb", "lib/vmpooler/providers/vsphere.rb"]
+    # although these files can come from any gem
+    def self.load_all_providers
+      self.new.load_from_gems
+    end
+
+    # @return [Array] - returns an array of gem names that contain a provider
+    def self.installed_providers
+      self.new.vmpooler_provider_gem_list.map(&:name)
+    end
+
+    # @return [Array] returns a list of vmpooler providers gem plugin specs
+    def vmpooler_provider_gem_list
+      gemspecs.find_all { |spec| File.directory?(File.join(spec.full_gem_path, provider_path)) } + included_lib_dirs
+    end
+
+    # Internal: Find any gems containing vmpooler provider plugins and load the main file in them.
+    #
+    # @return [Array[String]] - a array of provider files
+    # @param name [String] - the name of the provider to load
+    def load_from_gems(name = nil)
+      paths = gem_directories.map do |gem_path|
+        # we don't exactly know if the provider name matches the main file name that should be loaded
+        # so we use globs to get everything like the name
+        # this could mean that vsphere5 and vsphere6 are loaded when only vsphere5 is used
+        Dir.glob(File.join(gem_path, "*#{name}*.rb")).each do |file|
+          require file
+        end
+      end
+      paths.flatten
+    end
+
+    private
+
+    # @return [String] - the relative path to the vmpooler provider dir
+    # this is used when searching gems for this path
+    def provider_path
+      File.join('lib','vmpooler','providers')
+    end
+
+    # Add constants to array to skip over classes, ie. Vmpooler::PoolManager::Provider::Dummy
+    def excluded_classes
+      []
+    end
+
+    # paths to include in the search path
+    def included_lib_dirs
+      []
+    end
+
+    # returns an array of plugin classes by looking in the object space for all loaded classes
+    # that start with Vmpooler::PoolManager::Provider
+    def plugin_classes
+      unless @plugin_classes
+        load_plugins
+        # weed out any subclasses in the formatter
+        klasses = ObjectSpace.each_object(Class).find_all do |c|
+          c.name && c.name.split('::').count == 3 && c.name =~ /Vmpooler::PoolManager::Provider/
+        end
+        @plugin_classes = klasses - excluded_classes || []
+      end
+      @plugin_classes
+    end
+
+    def plugin_map
+      @plugin_map ||= Hash[plugin_classes.map { |gem| [gem.send(:name), gem] }]
+    end
+
+
+
+    # Internal: Retrieve a list of available gem paths from RubyGems.
+    #
+    # Returns an Array of Pathname objects.
+    def gem_directories
+      dirs = []
+      if has_rubygems?
+        dirs = gemspecs.map do |spec|
+          lib_path = File.expand_path(File.join(spec.full_gem_path,provider_path))
+          lib_path if File.exists? lib_path
+        end + included_lib_dirs
+      end
+      dirs.reject { |dir| dir.nil? }.uniq
+    end
+
+    # Internal: Check if RubyGems is loaded and available.
+    #
+    # Returns true if RubyGems is available, false if not.
+    def has_rubygems?
+      defined? ::Gem
+    end
+
+    # Internal: Retrieve a list of available gemspecs.
+    #
+    # Returns an Array of Gem::Specification objects.
+    def gemspecs
+      @gemspecs ||= if Gem::Specification.respond_to?(:latest_specs)
+                      Gem::Specification.latest_specs
+                    else
+                      Gem.searcher.init_gemspecs
+                    end
+    end
+
   end
 end

data/lib/vmpooler/version.rb

@@ -1,4 +1,3 @@
-
 module Vmpooler
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/lib/vmpooler.rb

@@ -11,7 +11,7 @@ module Vmpooler
   require 'yaml'
   require 'set'
 
-  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool providers].each do |lib|
+  %w[api graphite logger pool_manager statsd dummy_statsd generic_connection_pool].each do |lib|
     require "vmpooler/#{lib}"
   end
 
@@ -59,12 +59,12 @@ module Vmpooler
     parsed_config[:config]['clone_target'] = ENV['CLONE_TARGET'] if ENV['CLONE_TARGET']
     parsed_config[:config]['timeout'] = ENV['TIMEOUT'] if ENV['TIMEOUT']
     parsed_config[:config]['vm_lifetime_auth'] = ENV['VM_LIFETIME_AUTH'] if ENV['VM_LIFETIME_AUTH']
-    parsed_config[:config]['ssh_key'] = ENV['SSH_KEY'] if ENV['SSH_KEY']
     parsed_config[:config]['max_tries'] = ENV['MAX_TRIES'] if ENV['MAX_TRIES']
     parsed_config[:config]['retry_factor'] = ENV['RETRY_FACTOR'] if ENV['RETRY_FACTOR']
     parsed_config[:config]['create_folders'] = ENV['CREATE_FOLDERS'] if ENV['CREATE_FOLDERS']
     parsed_config[:config]['create_template_delta_disks'] = ENV['CREATE_TEMPLATE_DELTA_DISKS'] if ENV['CREATE_TEMPLATE_DELTA_DISKS']
     parsed_config[:config]['experimental_features'] = ENV['EXPERIMENTAL_FEATURES'] if ENV['EXPERIMENTAL_FEATURES']
+    parsed_config[:config]['purge_unconfigured_folders'] = ENV['PURGE_UNCONFIGURED_FOLDERS'] if ENV['PURGE_UNCONFIGURED_FOLDERS']
 
     parsed_config[:redis] = parsed_config[:redis] || {}
     parsed_config[:redis]['server'] = ENV['REDIS_SERVER'] || parsed_config[:redis]['server'] || 'localhost'
@@ -79,12 +79,14 @@ module Vmpooler
 
     parsed_config[:graphite] = parsed_config[:graphite] || {} if ENV['GRAPHITE_SERVER']
     parsed_config[:graphite]['server'] = ENV['GRAPHITE_SERVER'] if ENV['GRAPHITE_SERVER']
+    parsed_config[:graphite]['prefix'] = ENV['GRAPHITE_PREFIX'] if ENV['GRAPHITE_PREFIX']
+    parsed_config[:graphite]['port'] = ENV['GRAPHITE_PORT'] if ENV['GRAPHITE_PORT']
 
     parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
     if parsed_config.has_key? :auth
       parsed_config[:auth]['provider'] = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
       parsed_config[:auth][:ldap] = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
-      parsed_config[:auth][:ldap]['server'] = ENV['LDAP_SERVER'] if ENV['LDAP_SERVER']
+      parsed_config[:auth][:ldap]['host'] = ENV['LDAP_HOST'] if ENV['LDAP_HOST']
       parsed_config[:auth][:ldap]['port'] = ENV['LDAP_PORT'] if ENV['LDAP_PORT']
       parsed_config[:auth][:ldap]['base'] = ENV['LDAP_BASE'] if ENV['LDAP_BASE']
       parsed_config[:auth][:ldap]['user_object'] = ENV['LDAP_USER_OBJECT'] if ENV['LDAP_USER_OBJECT']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: vmpooler
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-17 00:00:00.000000000 Z
+date: 2018-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: puma