vmpooler 0.10.3 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1235ece5040bd685dafa1fb0d1d4a63b0002d80e58e2890d83b126d8a9ba51b
-  data.tar.gz: d9d623741face3692fc412b07737f29d5f0b727821d80e19277aa90c6936f504
+  metadata.gz: 19dd8ce83614c49e92f13e043c2ee7865f1c664d92aa8f13e47fc328eb033690
+  data.tar.gz: c3618c9c13591b1b06748ac3e5d1c2dc7ba2eb9793ea2f3f014a762fd6deb4ed
 SHA512:
-  metadata.gz: 409abfb3eb3e438a9e4d87a4fb95137b776d31a3d0ccad79b052589da56b052fad6df924a8fbf34eec5d2998878b409f149dfa8bd3d5665e490e3744af1f8fee
-  data.tar.gz: bcc2e4db8092b9b2c31bfb0327ee1394fa92f87d513dcee2488e114203bedd72f6a2cc8ef7dd99675014acc5cf6b72305fc21ac7a614bc57bf0dda9dbecc8835
+  metadata.gz: 8dcb8f8765f9c1fac52c2d0088aaa09a589b5145d5a81e901af614b8fcbbd588669b3f14e5c42f4d56d4681165f8814ef11b22073df6f97b85cef6fcb10d04e2
+  data.tar.gz: d7c917233daee973486a6a8add55222a92bb1708521caaefdf2666824c0785cebd1ce7e298448a66736bf7e0646f02a560067e54cc1d06a07ca6abae645898c9

data/bin/vmpooler

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'vmpooler'
 
@@ -49,6 +50,4 @@ if ENV['VMPOOLER_DEBUG']
   end
 end
 
-torun_threads.each do |th|
-  th.join
-end
+torun_threads.each(&:join)

data/lib/vmpooler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
   require 'date'
   require 'json'
@@ -27,13 +29,21 @@ module Vmpooler
       # Take the name of the config file either from an ENV variable or from the filepath argument
       config_file = ENV['VMPOOLER_CONFIG_FILE'] || filepath
       parsed_config = YAML.load_file(config_file) if File.exist? config_file
+      parsed_config[:config]['extra_config'] = ENV['EXTRA_CONFIG'] if ENV['EXTRA_CONFIG']
+      if parsed_config[:config]['extra_config']
+        extra_configs = parsed_config[:config]['extra_config'].split(',')
+        extra_configs.each do |config|
+          extra_config = YAML.load_file(config)
+          parsed_config.merge!(extra_config)
+        end
+      end
     end
 
     parsed_config ||= { config: {} }
 
     # Bail out if someone attempts to start vmpooler with dummy authentication
     # without enbaling debug mode.
-    if parsed_config.has_key? :auth
+    if parsed_config.key? :auth
       if parsed_config[:auth]['provider'] == 'dummy'
         unless ENV['VMPOOLER_DEBUG']
           warning = [
@@ -50,7 +60,7 @@ module Vmpooler
     parsed_config[:config]['task_limit'] = string_to_int(ENV['TASK_LIMIT']) || parsed_config[:config]['task_limit'] || 10
     parsed_config[:config]['migration_limit'] = string_to_int(ENV['MIGRATION_LIMIT']) if ENV['MIGRATION_LIMIT']
     parsed_config[:config]['vm_checktime'] = string_to_int(ENV['VM_CHECKTIME']) || parsed_config[:config]['vm_checktime'] || 1
-    parsed_config[:config]['vm_lifetime'] = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime']  || 24
+    parsed_config[:config]['vm_lifetime'] = string_to_int(ENV['VM_LIFETIME']) || parsed_config[:config]['vm_lifetime'] || 24
     parsed_config[:config]['prefix'] = ENV['PREFIX'] || parsed_config[:config]['prefix'] || ''
 
     parsed_config[:config]['logfile'] = ENV['LOGFILE'] if ENV['LOGFILE']
@@ -86,7 +96,7 @@ module Vmpooler
     parsed_config[:graphite]['port'] = string_to_int(ENV['GRAPHITE_PORT']) if ENV['GRAPHITE_PORT']
 
     parsed_config[:auth] = parsed_config[:auth] || {} if ENV['AUTH_PROVIDER']
-    if parsed_config.has_key? :auth
+    if parsed_config.key? :auth
       parsed_config[:auth]['provider'] = ENV['AUTH_PROVIDER'] if ENV['AUTH_PROVIDER']
       parsed_config[:auth][:ldap] = parsed_config[:auth][:ldap] || {} if parsed_config[:auth]['provider'] == 'ldap'
       parsed_config[:auth][:ldap]['host'] = ENV['LDAP_HOST'] if ENV['LDAP_HOST']
@@ -122,10 +132,8 @@ module Vmpooler
       end
     end
 
-    if parsed_config[:tagfilter]
-      parsed_config[:tagfilter].keys.each do |tag|
-        parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
-      end
+    parsed_config[:tagfilter]&.keys&.each do |tag|
+      parsed_config[:tagfilter][tag] = Regexp.new(parsed_config[:tagfilter][tag])
     end
 
     parsed_config[:uptime] = Time.now
@@ -171,7 +179,7 @@ module Vmpooler
   def self.pool_index(pools)
     pools_hash = {}
     index = 0
-    for pool in pools
+    pools.each do |pool|
       pools_hash[pool['name']] = index
       index += 1
     end
@@ -182,14 +190,15 @@ module Vmpooler
     # Returns a integer if input is a string
     return if s.nil?
     return unless s =~ /\d/
-    return Integer(s)
+
+    Integer(s)
   end
 
   def self.true?(obj)
-    obj.to_s.downcase == "true"
+    obj.to_s.downcase == 'true'
   end
 
-  def self.set_linked_clone(parsed_config)
+  def self.set_linked_clone(parsed_config) # rubocop:disable Naming/AccessorMethodName
     parsed_config[:config]['create_linked_clones'] = parsed_config[:config]['create_linked_clones'] || true
     parsed_config[:config]['create_linked_clones'] = ENV['CREATE_LINKED_CLONES'] if ENV['CREATE_LINKED_CLONES'] =~ /true|false/
     parsed_config[:config]['create_linked_clones'] = true?(parsed_config[:config]['create_linked_clones']) if parsed_config[:config]['create_linked_clones']

data/lib/vmpooler/api.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class API < Sinatra::Base
     def initialize

data/lib/vmpooler/api/dashboard.rb

@@ -1,7 +1,8 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class API
     class Dashboard < Sinatra::Base
-
       helpers do
         include Vmpooler::API::Helpers
       end
@@ -21,9 +22,11 @@ module Vmpooler
 
         if config[:graphs]
           return false unless config[:graphs]['server']
+
           @graph_server = config[:graphs]['server']
         elsif config[:graphite]
           return false unless config[:graphite]['server']
+
           @graph_server = config[:graphite]['server']
         else
           false
@@ -36,9 +39,11 @@ module Vmpooler
 
         if config[:graphs]
           return 'vmpooler' unless config[:graphs]['prefix']
+
           @graph_prefix = config[:graphs]['prefix']
         elsif config[:graphite]
           return false unless config[:graphite]['prefix']
+
           @graph_prefix = config[:graphite]['prefix']
         else
           false
@@ -48,12 +53,14 @@ module Vmpooler
       # what is the base URL for viewable graphs?
       def graph_url
         return false unless graph_server && graph_prefix
+
         @graph_url ||= "http://#{graph_server}/render?target=#{graph_prefix}"
       end
 
       # return a full URL to a viewable graph for a given metrics target (graphite syntax)
       def graph_link(target = '')
         return '' unless graph_url
+
         graph_url + target
       end
 
@@ -76,7 +83,7 @@ module Vmpooler
             history ||= {}
 
             begin
-              buffer = open(graph_link('.ready.*&from=-1hour&format=json')).read
+              buffer = URI.parse(graph_link('.ready.*&from=-1hour&format=json')).read
               history = JSON.parse(buffer)
 
               history.each do |pool|
@@ -100,7 +107,7 @@ module Vmpooler
                   end
                 end
               end
-            rescue
+            rescue StandardError
             end
           else
             pools.each do |pool|
@@ -129,7 +136,7 @@ module Vmpooler
         if params[:history]
           if graph_url
             begin
-              buffer = open(graph_link('.running.*&from=-1hour&format=json')).read
+              buffer = URI.parse(graph_link('.running.*&from=-1hour&format=json')).read
               JSON.parse(buffer).each do |pool|
                 if pool['target'] =~ /.*\.(.*)$/
                   pool['name'] = Regexp.last_match[1]
@@ -147,7 +154,7 @@ module Vmpooler
                   end
                 end
               end
-            rescue
+            rescue StandardError
             end
           end
         end

data/lib/vmpooler/api/helpers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
 
   class API
@@ -71,43 +73,44 @@ module Vmpooler
         )
 
         return true if ldap.bind
+
         return false
       end
 
       def authenticate(auth, username_str, password_str)
         case auth['provider']
-          when 'dummy'
-            return (username_str != password_str)
-          when 'ldap'
-            ldap_base = auth[:ldap]['base']
-            ldap_port = auth[:ldap]['port'] || 389
-
-            if ldap_base.is_a? Array
-              ldap_base.each do |search_base|
-                result = authenticate_ldap(
-                  ldap_port,
-                  auth[:ldap]['host'],
-                  auth[:ldap]['user_object'],
-                  search_base,
-                  username_str,
-                  password_str,
-                )
-                return true if result == true
-              end
-            else
+        when 'dummy'
+          return (username_str != password_str)
+        when 'ldap'
+          ldap_base = auth[:ldap]['base']
+          ldap_port = auth[:ldap]['port'] || 389
+          ldap_user_obj = auth[:ldap]['user_object']
+          ldap_host = auth[:ldap]['host']
+
+          unless ldap_base.is_a? Array
+            ldap_base = ldap_base.split
+          end
+
+          unless ldap_user_obj.is_a? Array
+            ldap_user_obj = ldap_user_obj.split
+          end
+
+          ldap_base.each do |search_base|
+            ldap_user_obj.each do |search_user_obj|
               result = authenticate_ldap(
                 ldap_port,
-                auth[:ldap]['host'],
-                auth[:ldap]['user_object'],
-                ldap_base,
+                ldap_host,
+                search_user_obj,
+                search_base,
                 username_str,
-                password_str,
+                password_str
               )
-              return result
+              return true if result
             end
-
-            return false
           end
+
+          return false
+        end
       end
 
       def export_tags(backend, hostname, tags)
@@ -124,6 +127,7 @@ module Vmpooler
 
         tags.each_pair do |tag, value|
           next unless filter = Vmpooler::API.settings.config[:tagfilter][tag]
+
           tags[tag] = value.match(filter).captures.join if value.match(filter)
         end
 
@@ -161,7 +165,7 @@ module Vmpooler
             backend.scard(key + pool['name'])
           end
         end
-        res.inject(0){ |m, x| m+x }.to_i
+        res.inject(0) { |m, x| m + x }.to_i
       end
 
       # Takes the pools and a key to run scard on
@@ -201,18 +205,18 @@ module Vmpooler
       def get_capacity_metrics(pools, backend)
         capacity = {
             current: 0,
-            total:   0,
+            total: 0,
             percent: 0
         }
 
         pools.each do |pool|
-          capacity[:total]   += pool['size'].to_i
+          capacity[:total] += pool['size'].to_i
         end
 
         capacity[:current] = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
 
         if capacity[:total] > 0
-          capacity[:percent] = ((capacity[:current].to_f / capacity[:total].to_f) * 100.0).round(1)
+          capacity[:percent] = (capacity[:current].fdiv(capacity[:total]) * 100.0).round(1)
         end
 
         capacity
@@ -220,16 +224,16 @@ module Vmpooler
 
       def get_queue_metrics(pools, backend)
         queue = {
-            pending:   0,
-            cloning:   0,
-            booting:   0,
-            ready:     0,
-            running:   0,
+            pending: 0,
+            cloning: 0,
+            booting: 0,
+            ready: 0,
+            running: 0,
             completed: 0,
-            total:     0
+            total: 0
         }
 
-        queue[:pending]   = get_total_across_pools_redis_scard(pools,'vmpooler__pending__', backend)
+        queue[:pending]   = get_total_across_pools_redis_scard(pools, 'vmpooler__pending__', backend)
         queue[:ready]     = get_total_across_pools_redis_scard(pools, 'vmpooler__ready__', backend)
         queue[:running]   = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
         queue[:completed] = get_total_across_pools_redis_scard(pools, 'vmpooler__completed__', backend)
@@ -306,11 +310,11 @@ module Vmpooler
         task = {
             duration: {
                 average: 0,
-                min:     0,
-                max:     0,
-                total:   0
+                min: 0,
+                max: 0,
+                total: 0
             },
-            count:    {
+            count: {
                 total: 0
             }
         }
@@ -450,7 +454,7 @@ module Vmpooler
       def pool_index(pools)
         pools_hash = {}
         index = 0
-        for pool in pools
+        pools.each do |pool|
           pools_hash[pool['name']] = index
           index += 1
         end
@@ -461,13 +465,14 @@ module Vmpooler
         prepared_template = backend.hget('vmpooler__template__prepared', pool['name'])
         return false if prepared_template.nil?
         return true if pool['template'] == prepared_template
+
         return false
       end
 
       def is_integer?(x)
         Integer(x)
         true
-      rescue
+      rescue StandardError
         false
       end
 
@@ -487,7 +492,7 @@ module Vmpooler
       def vm_ready?(vm_name, domain = nil)
         begin
           open_socket(vm_name, domain)
-        rescue => _err
+        rescue StandardError => _e
           return false
         end
 

data/lib/vmpooler/api/reroute.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class API
     class Reroute < Sinatra::Base

data/lib/vmpooler/api/v1.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Vmpooler
   class API
   class V1 < Sinatra::Base
@@ -37,22 +39,24 @@ module Vmpooler
     end
 
     def checkoutlock
-      Vmpooler::API::settings.checkoutlock
+      Vmpooler::API.settings.checkoutlock
     end
 
     def fetch_single_vm(template)
       template_backends = [template]
       aliases = Vmpooler::API.settings.config[:alias]
       if aliases
-        template_backends = template_backends + aliases[template] if aliases[template].is_a?(Array)
+        template_backends += aliases[template] if aliases[template].is_a?(Array)
         template_backends << aliases[template] if aliases[template].is_a?(String)
         pool_index = pool_index(pools)
         weighted_pools = {}
         template_backends.each do |t|
           next unless pool_index.key? t
+
           index = pool_index[t]
           clone_target = pools[index]['clone_target'] || config['clone_target']
           next unless config.key?('backend_weight')
+
           weight = config['backend_weight'][clone_target]
           if weight
             weighted_pools[t] = weight
@@ -75,11 +79,17 @@ module Vmpooler
         template_backends.each do |template_backend|
           vms = backend.smembers("vmpooler__ready__#{template_backend}")
           next if vms.empty?
+
           vms.reverse.each do |vm|
             ready = vm_ready?(vm, config['domain'])
             if ready
-              backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
-              return [vm, template_backend, template]
+              smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm)
+              if smoved
+                return [vm, template_backend, template]
+              else
+                metrics.increment("checkout.smove.failed.#{template_backend}")
+                return [nil, nil, nil]
+              end
             else
               backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm)
               metrics.increment("checkout.nonresponsive.#{template_backend}")
@@ -104,7 +114,7 @@ module Vmpooler
 
         backend.hset('vmpooler__vm__' + vm, 'token:token', request.env['HTTP_X_AUTH_TOKEN'])
         backend.hset('vmpooler__vm__' + vm, 'token:user',
-          backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user')
+                     backend.hget('vmpooler__token__' + request.env['HTTP_X_AUTH_TOKEN'], 'user')
         )
 
         if config['vm_lifetime_auth'].to_i > 0
@@ -136,14 +146,14 @@ module Vmpooler
             metrics.increment('checkout.empty.' + requested)
             break
           else
-            vms << [ vmpool, vmname, vmtemplate ]
+            vms << [vmpool, vmname, vmtemplate]
             metrics.increment('checkout.success.' + vmtemplate)
           end
         end
       end
 
       if failed
-        vms.each do |(vmpool, vmname, vmtemplate)|
+        vms.each do |(vmpool, vmname, _vmtemplate)|
           return_vm_to_ready_state(vmpool, vmname)
         end
         status 503
@@ -203,7 +213,7 @@ module Vmpooler
     def reset_pool(payload)
       result = { 'ok' => false }
 
-      payload.each do |poolname, count|
+      payload.each do |poolname, _count|
         backend.sadd('vmpooler__poolreset', poolname)
       end
       status 201
@@ -220,7 +230,7 @@ module Vmpooler
 
       payload.each do |poolname, clone_target|
         unless pools[pool_index[poolname]]['clone_target'] == clone_target
-          pools[pool_index[poolname]]['clone_target'] == clone_target
+          pools[pool_index[poolname]]['clone_target'] = clone_target
           backend.hset('vmpooler__config__clone_target', poolname, clone_target)
           pools_updated += 1
           status 201
@@ -234,42 +244,36 @@ module Vmpooler
     def sync_pool_templates
       pool_index = pool_index(pools)
       template_configs = backend.hgetall('vmpooler__config__template')
-      unless template_configs.nil?
-        template_configs.each do |poolname, template|
+      template_configs&.each do |poolname, template|
           if pool_index.include? poolname
             unless pools[pool_index[poolname]]['template'] == template
               pools[pool_index[poolname]]['template'] = template
             end
           end
-        end
       end
     end
 
     def sync_pool_sizes
       pool_index = pool_index(pools)
       poolsize_configs = backend.hgetall('vmpooler__config__poolsize')
-      unless poolsize_configs.nil?
-        poolsize_configs.each do |poolname, size|
+      poolsize_configs&.each do |poolname, size|
           if pool_index.include? poolname
             unless pools[pool_index[poolname]]['size'] == size.to_i
               pools[pool_index[poolname]]['size'] == size.to_i
             end
           end
-        end
       end
     end
 
     def sync_clone_targets
       pool_index = pool_index(pools)
       clone_target_configs = backend.hgetall('vmpooler__config__clone_target')
-      unless clone_target_configs.nil?
-        clone_target_configs.each do |poolname, clone_target|
+      clone_target_configs&.each do |poolname, clone_target|
           if pool_index.include? poolname
             unless pools[pool_index[poolname]]['clone_target'] == clone_target
               pools[pool_index[poolname]]['clone_target'] == clone_target
-              end
+            end
           end
-        end
       end
     end
 
@@ -368,36 +372,38 @@ module Vmpooler
       pending_hash = get_list_across_pools_redis_scard(pools, 'vmpooler__pending__', backend)
       lastBoot_hash = get_list_across_pools_redis_hget(pools, 'vmpooler__lastboot', backend)
 
-      pools.each do |pool|
-        # REMIND: move this out of the API and into the back-end
-        ready    = ready_hash[pool['name']]
-        running  = running_hash[pool['name']]
-        pending  = pending_hash[pool['name']]
-        max      = pool['size']
-        lastBoot = lastBoot_hash[pool['name']]
-        aka      = pool['alias']
-
-        result[:pools][pool['name']] = {
-          ready:    ready,
-          running:  running,
-          pending:  pending,
-          max:      max,
-          lastBoot: lastBoot
-        }
+      unless views and not views.include?("pools")
+        pools.each do |pool|
+          # REMIND: move this out of the API and into the back-end
+          ready    = ready_hash[pool['name']]
+          running  = running_hash[pool['name']]
+          pending  = pending_hash[pool['name']]
+          max      = pool['size']
+          lastBoot = lastBoot_hash[pool['name']]
+          aka      = pool['alias']
+
+          result[:pools][pool['name']] = {
+            ready: ready,
+            running: running,
+            pending: pending,
+            max: max,
+            lastBoot: lastBoot
+          }
 
-        if aka
-          result[:pools][pool['name']][:alias] = aka
-        end
+          if aka
+            result[:pools][pool['name']][:alias] = aka
+          end
 
-        # for backwards compatibility, include separate "empty" stats in "status" block
-        if ready == 0
-          result[:status][:empty] ||= []
-          result[:status][:empty].push(pool['name'])
+          # for backwards compatibility, include separate "empty" stats in "status" block
+          if ready == 0
+            result[:status][:empty] ||= []
+            result[:status][:empty].push(pool['name'])
 
-          result[:status][:ok] = false
-          result[:status][:message] = "Found #{result[:status][:empty].length} empty pools."
+            result[:status][:ok] = false
+            result[:status][:message] = "Found #{result[:status][:empty].length} empty pools."
+          end
         end
-      end unless views and not views.include?("pools")
+      end
 
       result[:status][:uptime] = (Time.now - Vmpooler::API.settings.config[:uptime]).round(1) if Vmpooler::API.settings.config[:uptime]
 
@@ -442,7 +448,6 @@ module Vmpooler
         if aka
           result[:pools][pool['name']][:alias] = aka
         end
-
       end
 
       ready_hash = get_list_across_pools_redis_scard(poolscopy, 'vmpooler__ready__', backend)
@@ -456,7 +461,7 @@ module Vmpooler
     get "#{api_prefix}/totalrunning/?" do
       content_type :json
       queue = {
-          running:   0,
+          running: 0
       }
 
       queue[:running] = get_total_across_pools_redis_scard(pools, 'vmpooler__running__', backend)
@@ -753,7 +758,7 @@ module Vmpooler
 
     def invalid_pool(payload)
       invalid = []
-      payload.each do |pool, clone_target|
+      payload.each do |pool, _clone_target|
         invalid << pool unless pool_exists?(pool)
       end
       invalid
@@ -837,7 +842,7 @@ module Vmpooler
         # Look up IP address of the hostname
         begin
           ipAddress = TCPSocket.gethostbyname(params[:hostname])[3]
-        rescue
+        rescue StandardError
           ipAddress = ""
         end
 
@@ -874,6 +879,8 @@ module Vmpooler
 
           status 200
           result['ok'] = true
+        else
+          metrics.increment('delete.srem.failed')
         end
       end
 
@@ -893,7 +900,7 @@ module Vmpooler
       if backend.exists('vmpooler__vm__' + params[:hostname])
         begin
           jdata = JSON.parse(request.body.read)
-        rescue
+        rescue StandardError
           halt 400, JSON.pretty_generate(result)
         end
 
@@ -909,13 +916,6 @@ module Vmpooler
                 max_lifetime_upper_limit = max_lifetime_upper_limit.to_i
                 if arg.to_i >= max_lifetime_upper_limit
                   failure.push("You provided a lifetime (#{arg}) that exceeds the configured maximum of #{max_lifetime_upper_limit}.")
-                else
-                  # also make sure we do not extend past max_lifetime_upper_limit
-                  rdata = backend.hgetall('vmpooler__vm__' + params[:hostname])
-                  running = ((Time.now - Time.parse(rdata['checkout'])) / 60 / 60).round(2)
-                  unless running + arg.to_i < max_lifetime_upper_limit
-                    failure.push("You provided a lifetime (#{arg}) that will extend the current lifetime past the configured maximum of #{max_lifetime_upper_limit}.")
-                  end
                 end
               end
 
@@ -936,7 +936,7 @@ module Vmpooler
           end
         end
 
-        if failure.size > 0
+        if !failure.empty?
           status 400
           result['failure'] = failure
         else